bump owasp job version

.ds.baseline

@@ -1,388 +0,0 @@
-{
-  "version": "1.5.0",
-  "plugins_used": [
-    {
-      "name": "ArtifactoryDetector"
-    },
-    {
-      "name": "AWSKeyDetector"
-    },
-    {
-      "name": "AzureStorageKeyDetector"
-    },
-    {
-      "name": "Base64HighEntropyString",
-      "limit": 4.5
-    },
-    {
-      "name": "BasicAuthDetector"
-    },
-    {
-      "name": "CloudantDetector"
-    },
-    {
-      "name": "DiscordBotTokenDetector"
-    },
-    {
-      "name": "GitHubTokenDetector"
-    },
-    {
-      "name": "GitLabTokenDetector"
-    },
-    {
-      "name": "HexHighEntropyString",
-      "limit": 3.0
-    },
-    {
-      "name": "IbmCloudIamDetector"
-    },
-    {
-      "name": "IbmCosHmacDetector"
-    },
-    {
-      "name": "IPPublicDetector"
-    },
-    {
-      "name": "JwtTokenDetector"
-    },
-    {
-      "name": "KeywordDetector",
-      "keyword_exclude": ""
-    },
-    {
-      "name": "MailchimpDetector"
-    },
-    {
-      "name": "NpmDetector"
-    },
-    {
-      "name": "OpenAIDetector"
-    },
-    {
-      "name": "PrivateKeyDetector"
-    },
-    {
-      "name": "PypiTokenDetector"
-    },
-    {
-      "name": "SendGridDetector"
-    },
-    {
-      "name": "SlackDetector"
-    },
-    {
-      "name": "SoftlayerDetector"
-    },
-    {
-      "name": "SquareOAuthDetector"
-    },
-    {
-      "name": "StripeDetector"
-    },
-    {
-      "name": "TelegramBotTokenDetector"
-    },
-    {
-      "name": "TwilioKeyDetector"
-    }
-  ],
-  "filters_used": [
-    {
-      "path": "detect_secrets.filters.allowlist.is_line_allowlisted"
-    },
-    {
-      "path": "detect_secrets.filters.common.is_baseline_file",
-      "filename": ".ds.baseline"
-    },
-    {
-      "path": "detect_secrets.filters.common.is_ignored_due_to_verification_policies",
-      "min_level": 2
-    },
-    {
-      "path": "detect_secrets.filters.heuristic.is_indirect_reference"
-    },
-    {
-      "path": "detect_secrets.filters.heuristic.is_likely_id_string"
-    },
-    {
-      "path": "detect_secrets.filters.heuristic.is_lock_file"
-    },
-    {
-      "path": "detect_secrets.filters.heuristic.is_not_alphanumeric_string"
-    },
-    {
-      "path": "detect_secrets.filters.heuristic.is_potential_uuid"
-    },
-    {
-      "path": "detect_secrets.filters.heuristic.is_prefixed_with_dollar_sign"
-    },
-    {
-      "path": "detect_secrets.filters.heuristic.is_sequential_string"
-    },
-    {
-      "path": "detect_secrets.filters.heuristic.is_swagger_file"
-    },
-    {
-      "path": "detect_secrets.filters.heuristic.is_templated_secret"
-    }
-  ],
-  "results": {
-    ".github/workflows/checks.yml": [
-      {
-        "type": "Secret Keyword",
-        "filename": ".github/workflows/checks.yml",
-        "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
-        "is_verified": false,
-        "line_number": 27,
-        "is_secret": false
-      },
-      {
-        "type": "Basic Auth Credentials",
-        "filename": ".github/workflows/checks.yml",
-        "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
-        "is_verified": false,
-        "line_number": 44,
-        "is_secret": false
-      }
-    ],
-    ".github/workflows/daily_checks.yml": [
-      {
-        "type": "Secret Keyword",
-        "filename": ".github/workflows/daily_checks.yml",
-        "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
-        "is_verified": false,
-        "line_number": 61,
-        "is_secret": false
-      },
-      {
-        "type": "Basic Auth Credentials",
-        "filename": ".github/workflows/daily_checks.yml",
-        "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
-        "is_verified": false,
-        "line_number": 77,
-        "is_secret": false
-      }
-    ],
-    "app/enums.py": [
-      {
-        "type": "Secret Keyword",
-        "filename": "app/enums.py",
-        "hashed_secret": "12322e07b94ee3c7cd65a2952ece441538b53eb3",
-        "is_verified": false,
-        "line_number": 123,
-        "is_secret": false
-      }
-    ],
-    "app/notifications/receive_notifications.py": [
-      {
-        "type": "Base64 High Entropy String",
-        "filename": "app/notifications/receive_notifications.py",
-        "hashed_secret": "d70eab08607a4d05faa2d0d6647206599e9abc65",
-        "is_verified": false,
-        "line_number": 29,
-        "is_secret": false
-      }
-    ],
-    "deploy-config/sandbox.yml": [
-      {
-        "type": "Secret Keyword",
-        "filename": "deploy-config/sandbox.yml",
-        "hashed_secret": "113151dd10316fcb0d5507b6215d78e2f3fe9e54",
-        "is_verified": false,
-        "line_number": 11,
-        "is_secret": false
-      }
-    ],
-    "sample.env": [
-      {
-        "type": "Basic Auth Credentials",
-        "filename": "sample.env",
-        "hashed_secret": "5b98cf4c3d794c8af1fcd7991e89cd4e52fb42a4",
-        "is_verified": false,
-        "line_number": 16,
-        "is_secret": false
-      }
-    ],
-    "tests/app/aws/test_s3.py": [
-      {
-        "type": "Hex High Entropy String",
-        "filename": "tests/app/aws/test_s3.py",
-        "hashed_secret": "67a74306b06d0c01624fe0d0249a570f4d093747",
-        "is_verified": false,
-        "line_number": 24,
-        "is_secret": false
-      }
-    ],
-    "tests/app/clients/test_document_download.py": [
-      {
-        "type": "Secret Keyword",
-        "filename": "tests/app/clients/test_document_download.py",
-        "hashed_secret": "3acfb2c2b433c0ea7ff107e33df91b18e52f960f",
-        "is_verified": false,
-        "line_number": 14,
-        "is_secret": false
-      }
-    ],
-    "tests/app/clients/test_performance_platform.py": [
-      {
-        "type": "Base64 High Entropy String",
-        "filename": "tests/app/clients/test_performance_platform.py",
-        "hashed_secret": "76bb66c38ac4046bf73cd4a2c35a2b0af94aeb61",
-        "is_verified": false,
-        "line_number": 84,
-        "is_secret": false
-      }
-    ],
-    "tests/app/dao/test_services_dao.py": [
-      {
-        "type": "Secret Keyword",
-        "filename": "tests/app/dao/test_services_dao.py",
-        "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
-        "is_verified": false,
-        "line_number": 265,
-        "is_secret": false
-      }
-    ],
-    "tests/app/dao/test_users_dao.py": [
-      {
-        "type": "Secret Keyword",
-        "filename": "tests/app/dao/test_users_dao.py",
-        "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
-        "is_verified": false,
-        "line_number": 52,
-        "is_secret": false
-      },
-      {
-        "type": "Secret Keyword",
-        "filename": "tests/app/dao/test_users_dao.py",
-        "hashed_secret": "f2c57870308dc87f432e5912d4de6f8e322721ba",
-        "is_verified": false,
-        "line_number": 176,
-        "is_secret": false
-      }
-    ],
-    "tests/app/db.py": [
-      {
-        "type": "Secret Keyword",
-        "filename": "tests/app/db.py",
-        "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
-        "is_verified": false,
-        "line_number": 87,
-        "is_secret": false
-      }
-    ],
-    "tests/app/notifications/test_receive_notification.py": [
-      {
-        "type": "Secret Keyword",
-        "filename": "tests/app/notifications/test_receive_notification.py",
-        "hashed_secret": "913a73b565c8e2c8ed94497580f619397709b8b6",
-        "is_verified": false,
-        "line_number": 24,
-        "is_secret": false
-      },
-      {
-        "type": "Base64 High Entropy String",
-        "filename": "tests/app/notifications/test_receive_notification.py",
-        "hashed_secret": "d70eab08607a4d05faa2d0d6647206599e9abc65",
-        "is_verified": false,
-        "line_number": 54,
-        "is_secret": false
-      }
-    ],
-    "tests/app/notifications/test_validators.py": [
-      {
-        "type": "Base64 High Entropy String",
-        "filename": "tests/app/notifications/test_validators.py",
-        "hashed_secret": "6c1a8443963d02d13ffe575a71abe19ea731fb66",
-        "is_verified": false,
-        "line_number": 768,
-        "is_secret": false
-      }
-    ],
-    "tests/app/service/test_rest.py": [
-      {
-        "type": "Secret Keyword",
-        "filename": "tests/app/service/test_rest.py",
-        "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
-        "is_verified": false,
-        "line_number": 1274,
-        "is_secret": false
-      }
-    ],
-    "tests/app/test_cloudfoundry_config.py": [
-      {
-        "type": "Secret Keyword",
-        "filename": "tests/app/test_cloudfoundry_config.py",
-        "hashed_secret": "e5e178db7317356946d13e5d2da037d39ac61c71",
-        "is_verified": false,
-        "line_number": 12,
-        "is_secret": false
-      },
-      {
-        "type": "Basic Auth Credentials",
-        "filename": "tests/app/test_cloudfoundry_config.py",
-        "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
-        "is_verified": false,
-        "line_number": 14,
-        "is_secret": false
-      },
-      {
-        "type": "Secret Keyword",
-        "filename": "tests/app/test_cloudfoundry_config.py",
-        "hashed_secret": "cfd48edeb81ba7d48cbddcf1eeede25ba67057e8",
-        "is_verified": false,
-        "line_number": 33,
-        "is_secret": false
-      }
-    ],
-    "tests/app/user/test_rest.py": [
-      {
-        "type": "Secret Keyword",
-        "filename": "tests/app/user/test_rest.py",
-        "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
-        "is_verified": false,
-        "line_number": 106,
-        "is_secret": false
-      },
-      {
-        "type": "Secret Keyword",
-        "filename": "tests/app/user/test_rest.py",
-        "hashed_secret": "0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33",
-        "is_verified": false,
-        "line_number": 962,
-        "is_secret": false
-      }
-    ],
-    "tests/notifications_utils/clients/antivirus/test_antivirus_client.py": [
-      {
-        "type": "Secret Keyword",
-        "filename": "tests/notifications_utils/clients/antivirus/test_antivirus_client.py",
-        "hashed_secret": "932b25270abe1301c22c709a19082dff07d469ff",
-        "is_verified": false,
-        "line_number": 16,
-        "is_secret": false
-      }
-    ],
-    "tests/notifications_utils/clients/encryption/test_encryption_client.py": [
-      {
-        "type": "Secret Keyword",
-        "filename": "tests/notifications_utils/clients/encryption/test_encryption_client.py",
-        "hashed_secret": "f1e923a9667de11be6a210849a8651c1bfd81605",
-        "is_verified": false,
-        "line_number": 13,
-        "is_secret": false
-      }
-    ],
-    "tests/notifications_utils/clients/zendesk/test_zendesk_client.py": [
-      {
-        "type": "Secret Keyword",
-        "filename": "tests/notifications_utils/clients/zendesk/test_zendesk_client.py",
-        "hashed_secret": "913a73b565c8e2c8ed94497580f619397709b8b6",
-        "is_verified": false,
-        "line_number": 16,
-        "is_secret": false
-      }
-    ]
-  },
-  "generated_at": "2024-07-10T20:12:22Z"
-}

.github/ISSUE_TEMPLATE/create-a-new-adr-from-template.md

@@ -0,0 +1,125 @@
+---
+name: Create a new ADR from template
+about: Use this template to create a new Architectural Decision Record.
+title: ADR
+labels: documentation
+assignees: ''
+
+---
+
+# TITLE:  ADR Title Here
+
+
+| CREATED DATE | LAST UPDATED | STATUS | IMPLEMENTED | AUTHOR | STAKEHOLDERS |
+| :---: | :---: | :---: | :---: | :---: |
+| Date when ADR was created - MM/DD/YYYY format | Date when ADR was last updated - MM/DD/YYYY format - or N/A | Current ADR status - one of Proposed /  Accepted / Rejected / Deprecated / Superseded By ADR - link to ADR | Yes or No | GitHub username(s) of author(s) | GitHub username(s) or team name(s) of other folks involved |
+
+
+## CONTEXT AND PROBLEM STATEMENT
+
+**OPEN ISSUE(S):** Link(s) to related issue(s) here, if applicable.
+
+Explain the nature of the problem, challenge, or decision to be made here. Be as
+succinct as possible, but don't leave out any details.  State things matter of
+factly and as objectively as you can.
+
+
+## DECISION DRIVERS
+
+List anything that plays a major role in making a decision here.  These could
+be one or more of the following:
+
+- Desired qualities
+- Desired outcomes
+- Primary concerns
+- Constraints
+- ... (anything else pertinent to the decision-making process)
+
+
+### SECURITY COMPLIANCE CONSIDERATIONS
+
+Because we work in a regulated space with many compliance requirements, we need
+to make sure we're accounting for any security concerns and adhering to all
+security compliance requirements.  List them in this section along with any
+relevant details:
+
+- Security concern 1
+  - Concern detail 1
+  - Concern detail 2
+  - Concern detail ...
+
+- Security concern 2
+  - Concern detail 1
+  - Concern detail 2
+  - Concern detail ...
+
+
+## CONSIDERED OPTIONS
+
+List all options that have either been discussed or thought of as a potential
+solution to the context and problem statement.  Include any pros and cons with
+each option, like so:
+
+- **Name of first option:**  A brief summary of the option.
+  - Pros:
+    - Pro 1
+    - Pro 2
+    - Pro ...
+
+  - Cons:
+    - Con 1
+    - Con 2
+    - Con ...
+
+- **Name of second option:**  A brief summary of the option.
+  - Pros:
+    - Pro 1
+    - Pro 2
+    - Pro ...
+
+  - Cons:
+    - Con 1
+    - Con 2
+    - Con ...
+
+
+## PROPOSED OR CHOSEN OPTION:  Proposed/Chosen Option Title Here
+
+Summarize the decision for the proposed/chosen option here.  Be as concise and
+objective as possible while including all relevant details so that a clear
+justification is provided.  Include a list of consequences for choosing this
+option, both positive and negative:
+
+
+### Consequences
+
+- Positive
+  - Positive consequence 1
+  - Positive consequence 2
+  - Positive consequence ...
+
+- Negative
+  - Negative consequence 1
+  - Negative consequence 2
+  - Negative consequence ...
+
+
+## VALIDATION AND NEXT STEPS
+
+This section likely won't be filled out until a decision has been made by the
+team and the ADR is accepted.  If this comes to pass, then write up the criteria
+that would ensure this ADR is both implemented and tested correctly and
+adequately.  This could be a short summary and/or a list of things:
+
+- **Criterion name 1:**  Description of criterion 1
+  - Requirement or action 1
+  - Requirement or action 2
+  - Requirement or action ...
+
+- **Criterion name 2:**  Description of criterion 2
+  - Requirement or action 1
+  - Requirement or action 2
+  - Requirement or action ...
+
+Lastly, include a link(s) to an issue(s) that represent the work that will
+take place as follow-ups to this ADR.

.github/ISSUE_TEMPLATE/create-new-adr-form.yml

@@ -1,41 +1,59 @@
 name: Create a new ADR
-description: Create a new issue with an ADR proposal
-labels: ["ADR: proposed", "documentation"]
+description: Create a new Architectural Decision Record.
+title: "[ADR]: "
+labels: ["documentation"]
 body:
   - type: markdown
     attributes:
       value: |
         Use this form to create a new Architectural Decision Record.
-  - type: textarea
-    id: context
+  - type: input
+    id: title
+    attributes:
+      label: ADR Title
+      description: What is the title of this ADR?
+      placeholder: e.g., Establishing ADRs for US Notify
     validations:
       required: true
+  - type: input
+    id: created_date
     attributes:
-      label: Context
-      description: >
-        Explain the nature of the problem, challenge, or decision to be made here.
-        Be as succinct as possible, but do not leave out any details.  State things
-        matter of factly and as objectively as you can.
-  - type: textarea
-    id: decision
+      label: Created Date
+      description: What is the date this ADR is being created?
+      placeholder: MM/DD/YYYY
     validations:
       required: true
+  - type: dropdown
+    id: status
     attributes:
-      label: Decision
-      description: >
-        What was decided? This should be an active-voice statement. For example,
-        "We will use the US Web Design System." Note any major options that were
-        not selected.
-  - type: textarea
-    id: consequences
+      label: Status
+      description: What is the current status of this ADR?
+      options:
+        - Proposed
+        - Accepted
+        - Rejected
+        - Deprecated
+        - Superseded By
     validations:
       required: true
+  - type: dropdown
+    id: implemented
     attributes:
-      label: Consequences
-      description: >
-        What are the consequences of this decision? Consequences can be
-        positive, negative, or neutral, but all known consequences of this
-        decision should be listed here.
+      label: Implemented
+      description: Is this ADR implemented?
+      options:
+        - 'Yes'
+        - 'No'
+    validations:
+      required: true
+  - type: input
+    id: superseded_by
+    attributes:
+      label: Superseded By
+      description: If this ADR is superseded by another, which one is it?
+      placeholder: e.g., ADR-0001
+    validations:
+      required: false
   - type: input
     id: author
     attributes:
@@ -52,12 +70,61 @@ body:
       placeholder: Enter their GitHub username(s) here
     validations:
       required: false
-  - type: textarea
-    id: next-steps
+  - type: input
+    id: linked-github-issue
     attributes:
-      label: Next Steps
-      description: >
-        Once option has already been chosen and justified, summarize or list the next steps
-        that came from the decision. Link to GitHub issues that track these steps.
+      label: Related Issue(s)
+      description: List any related GitHub issues to this ADR here (issue number only).
+      placeholder: e.g., 288
+    validations:
+      required: false
+  - type: textarea
+    id: context-and-problem-statement
+    attributes:
+      label: Context and Problem Statement
+      description: Explain the nature of the problem, challenge, or decision to be made here.
+      placeholder: Be as succinct as possible, but do not leave out any details.  State things matter of factly and as objectively as you can.
+    validations:
+      required: true
+  - type: textarea
+    id: decision-drivers
+    attributes:
+      label: Decision Drivers
+      description: List anything that plays a major role in making a decision here.
+    validations:
+      required: true
+  - type: textarea
+    id: security-compliance-considerations
+    attributes:
+      label: Security Compliance Considerations
+      description: List any security concerns in this section along with any relevant details.
+    validations:
+      required: true
+  - type: textarea
+    id: considered-options
+    attributes:
+      label: Considered Options
+      description: List all options that have either been discussed or thought of as a potential solution to the context and problem statement.  Include any pros and cons with each option.
+    validations:
+      required: true
+  - type: input
+    id: proposed-option
+    attributes:
+      label: Proposed Option
+      description: If an option is already clear or has been chosen, please name it here.
+    validations:
+      required: false
+  - type: textarea
+    id: proposed-option-justification
+    attributes:
+      label: Proposed or Chosen Option
+      description: Summarize the decision for the proposed/chosen option here, including a list of positive and negative consequences.
+    validations:
+      required: false
+  - type: textarea
+    id: validation-and-next-steps
+    attributes:
+      label: Validation and Next Steps
+      description: If an option has already been chosen and justified, summarize or list the validation and next steps that came from the decision.
     validations:
       required: false

.github/actions/setup-project/action.yml

@@ -9,10 +9,10 @@ runs:
         sudo apt-get update \
         && sudo apt-get install -y --no-install-recommends \
         libcurl4-openssl-dev
-    - name: Set up Python 3.12.3
-      uses: actions/setup-python@v4
+    - name: Set up Python 3.9
+      uses: actions/setup-python@v3
       with:
-        python-version: "3.12.3"
-    - name: Install poetry
+        python-version: "3.9"
+    - name: Install pipenv
       shell: bash
-      run: pip install --upgrade poetry
+      run: pip install --upgrade pipenv

.github/dependabot.yml

@@ -9,3 +9,4 @@ updates:
     directory: "/" # Location of package manifests
     schedule:
       interval: "daily"
+

.github/pull_request_template.md

@@ -1,22 +0,0 @@
-<!--
-Not sure what you should include or write in a pull request?  Please read the
-[pull request documentation in our docs!](https://github.com/GSA/notifications-api/blob/main/docs/all.md#pull-requests)
--->
-
-*A note to PR reviewers: it may be helpful to review our [code review documentation](https://github.com/GSA/notifications-api/blob/main/docs/all.md#code-reviews) to know what to keep in mind while reviewing pull requests.*
-
-## Description
-
-Please enter a detailed description here.
-
-## TODO (optional)
-
-* [ ] TODO item 1
-* [ ] TODO item 2
-* [ ] TODO item ...
-
-## Security Considerations
-
-* Consideration 1
-* Consideration 2
-* Consideration ...

.github/workflows/adr-accepted.yml

@@ -1,83 +0,0 @@
-name: ADR accepted
-
-on:
-  issues:
-    types:
-      - closed
-
-permissions:
-  contents: read
-
-jobs:
-  accept:
-    runs-on: ubuntu-latest
-    steps:
-
-    - name: check for tags
-      if: "${{ !contains(github.event.issue.labels.*.name, 'ADR: accepted' )}}"
-      shell: bash
-      run: exit 0
-
-    - name: checkout main branch
-      uses: actions/checkout@v4
-      with:
-        ref: main
-        ssh-key: ${{ secrets.SSH_PRIVATE_KEY }}
-
-    - name: get ADR number
-      id: next
-      shell: bash
-      run: |
-        mkdir -p docs/adrs
-        LAST_ADR=$(ls docs/adrs/*.md | grep -Eo "/[0-9]+-" | sort | tail -n1 | grep -Eo "[0-9]+")
-        LAST_ADR=$(echo "$LAST_ADR" | sed -E 's/^0+//')
-        NEXT_ADR=$(($LAST_ADR + 1))
-        NEXT_ADR=$(printf "%04i" "$NEXT_ADR")
-        echo "number=$NEXT_ADR" >> "$GITHUB_OUTPUT"
-
-    - name: get date
-      id: date
-      shell: bash
-      run: echo "date=$(date +'%B %d, %Y')" >> "$GITHUB_OUTPUT"
-
-    - name: build filename
-      id: filename
-      shell: bash
-      run: |
-        SLUG=$(printf '%q\n' "${{ github.event.issue.title }}" | tr A-Z a-z)
-        SLUG=$(printf '%q\n' "$SLUG" | iconv -c -t ascii//TRANSLIT)
-        SLUG=$(printf '%q\n' "$SLUG" | sed -E 's/[^a-z0-9]+/-/g' | sed -E 's/-+/-/g' | sed -E 's/^-+|-+$//g')
-
-        FILENAME="docs/adrs/${{ steps.next.outputs.number }}-$SLUG.md"
-        echo "slug=$SLUG" >> "$GITHUB_OUTPUT"
-        echo "filename=$FILENAME" >> "$GITHUB_OUTPUT"
-
-    - name: write the ADR
-      uses: DamianReeves/write-file-action@v1.2
-      with:
-        path: ${{ steps.filename.outputs.filename }}
-        write-mode: overwrite
-        contents: |
-          # ${{ github.event.issue.title }}
-
-          Status: Accepted
-          Date: ${{ steps.date.outputs.date }}
-
-          ${{ github.event.issue.body }}
-
-    - name: branch, commit, and open PR
-      shell: bash
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      run: |
-        BRANCH="adr/auto/${{ steps.filename.outputs.slug }}"
-        git config --global user.email "tts@gsa.gov"
-        git config --global user.name "Notify ADR Automation"
-        git checkout -b $BRANCH
-        git add docs/adrs/*.md
-        git commit -m "add ADR ${{ steps.next.outputs.number }}: ${{ github.event.issue.title }}"
-        git push -f origin $BRANCH
-        gh pr create \
-          --title "Add ADR ${{ steps.next.outputs.number }} to the repo" \
-          --body "This pull request was opened automatically because #${{ github.event.issue.number }} was closed after being marked as an approved ADR. It contains a markdown file capturing the ADR body at the time the issue was closed. Please verify that the markdown is correct before merging!" || true
-        gh pr merge $BRANCH --auto --squash || true

.github/workflows/checks.yml

@@ -36,64 +36,53 @@ jobs:
           - 5432:5432
 
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v3
     - uses: ./.github/actions/setup-project
     - name: Install application dependencies
       run: make bootstrap
       env:
         SQLALCHEMY_DATABASE_TEST_URI: postgresql://user:password@localhost:5432/test_notification_api
-        NOTIFY_E2E_TEST_EMAIL: ${{ secrets.NOTIFY_E2E_TEST_EMAIL }}
-        NOTIFY_E2E_TEST_HTTP_AUTH_PASSWORD: ${{ secrets.NOTIFY_E2E_TEST_HTTP_AUTH_PASSWORD }}
-        NOTIFY_E2E_TEST_HTTP_AUTH_USER: ${{ secrets.NOTIFY_E2E_TEST_HTTP_AUTH_USER }}
-        NOTIFY_E2E_TEST_PASSWORD: ${{ secrets.NOTIFY_E2E_TEST_PASSWORD }}
     - name: Run style checks
-      run: poetry run flake8 .
+      run: pipenv run flake8 .
     - name: Check imports alphabetized
-      run: poetry run isort --check-only ./app ./tests
-    - name: Check for dead code
-      run: make dead-code
+      run: pipenv run isort --check-only ./app ./tests
     - name: Run tests with coverage
-      run: poetry run coverage run --omit=*/notifications_utils/*,*/migrations/* -m pytest --maxfail=10
+      run: pipenv run coverage run --omit=*/notifications_utils/* -m pytest -n4 --maxfail=10
       env:
         SQLALCHEMY_DATABASE_TEST_URI: postgresql://user:password@localhost:5432/test_notification_api
-        NOTIFY_E2E_TEST_EMAIL: ${{ secrets.NOTIFY_E2E_TEST_EMAIL }}
-        NOTIFY_E2E_TEST_HTTP_AUTH_PASSWORD: ${{ secrets.NOTIFY_E2E_TEST_HTTP_AUTH_PASSWORD }}
-        NOTIFY_E2E_TEST_HTTP_AUTH_USER: ${{ secrets.NOTIFY_E2E_TEST_HTTP_AUTH_USER }}
-        NOTIFY_E2E_TEST_PASSWORD: ${{ secrets.NOTIFY_E2E_TEST_PASSWORD }}
     - name: Check coverage threshold
-      # TODO get this back up to 95
-      run: poetry run coverage report --fail-under=95
+      run: pipenv run coverage report --fail-under=50
 
   validate-new-relic-config:
     runs-on: ubuntu-latest
     environment: staging
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
       - uses: ./.github/actions/setup-project
-      - name: Install poetry packages
-        run: poetry install
+      - name: Install pipenv packages
+        run: pipenv install --dev
       - name: Validate NewRelic config
         env:
           NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY }}
           # Need to set a NEW_RELIC_ENVIRONMENT with monitor_mode: true
           NEW_RELIC_ENVIRONMENT: staging
-        run: poetry run newrelic-admin validate-config $NEW_RELIC_CONFIG_FILE
+        run: pipenv run newrelic-admin validate-config $NEW_RELIC_CONFIG_FILE
 
   pip-audit:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
       - uses: ./.github/actions/setup-project
       - name: Create requirements.txt
-        run: poetry export --without-hashes --format=requirements.txt > requirements.txt
-      - uses: pypa/gh-action-pip-audit@v1.0.8
+        run: pipenv requirements > requirements.txt
+      - uses: pypa/gh-action-pip-audit@v1.0.6
         with:
           inputs: requirements.txt
 
   static-scan:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
       - uses: ./.github/actions/setup-project
       - name: Install bandit
         run: pip install bandit
@@ -118,24 +107,20 @@ jobs:
           # Maps tcp port 5432 on service container to the host
           - 5432:5432
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
       - uses: ./.github/actions/setup-project
       - name: Install application dependencies
         run: make bootstrap
         env:
           SQLALCHEMY_DATABASE_TEST_URI: postgresql://user:password@localhost:5432/test_notification_api
-          NOTIFY_E2E_TEST_EMAIL: ${{ secrets.NOTIFY_E2E_TEST_EMAIL }}
-          NOTIFY_E2E_TEST_HTTP_AUTH_PASSWORD: ${{ secrets.NOTIFY_E2E_TEST_HTTP_AUTH_PASSWORD }}
-          NOTIFY_E2E_TEST_HTTP_AUTH_USER: ${{ secrets.NOTIFY_E2E_TEST_HTTP_AUTH_USER }}
-          NOTIFY_E2E_TEST_PASSWORD: ${{ secrets.NOTIFY_E2E_TEST_PASSWORD }}
       - name: Run server
         run: make run-flask &
         env:
           SQLALCHEMY_DATABASE_TEST_URI: postgresql://user:password@localhost:5432/test_notification_api
-      - name: Run OWASP API Scan
-        uses: zaproxy/action-api-scan@v0.5.0
+      - name: Run OWASP Baseline Scan
+        uses: zaproxy/action-api-scan@v0.3.1
         with:
-          docker_name: 'ghcr.io/zaproxy/zaproxy:weekly'
+          docker_name: 'owasp/zap2docker-stable'
           target: 'http://localhost:6011/docs/openapi.yml'
           fail_action: true
           allow_issue_writing: false

.github/workflows/codeql.yml

@@ -1,93 +0,0 @@
-# For most projects, this workflow file will not need changing; you simply need
-# to commit it to your repository.
-#
-# You may wish to alter this file to override the set of languages analyzed,
-# or to provide custom queries or build logic.
-#
-# ******** NOTE ********
-# We have attempted to detect the languages in your repository. Please check
-# the `language` matrix defined below to confirm you have the correct set of
-# supported CodeQL languages.
-#
-name: "CodeQL"
-
-on:
-  push:
-    branches: [ "main", "production" ]
-  pull_request:
-    branches: [ "main", "production" ]
-  schedule:
-    - cron: '15 8 * * 2'
-
-jobs:
-  analyze:
-    name: Analyze (${{ matrix.language }})
-    # Runner size impacts CodeQL analysis time. To learn more, please see:
-    #   - https://gh.io/recommended-hardware-resources-for-running-codeql
-    #   - https://gh.io/supported-runners-and-hardware-resources
-    #   - https://gh.io/using-larger-runners (GitHub.com only)
-    # Consider using larger runners or machines with greater resources for possible analysis time improvements.
-    runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }}
-    timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }}
-    permissions:
-      # required for all workflows
-      security-events: write
-
-      # required to fetch internal or private CodeQL packs
-      packages: read
-
-      # only required for workflows in private repositories
-      actions: read
-      contents: read
-
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-        - language: python
-          build-mode: none
-        # CodeQL supports the following values keywords for 'language': 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift'
-        # Use `c-cpp` to analyze code written in C, C++ or both
-        # Use 'java-kotlin' to analyze code written in Java, Kotlin or both
-        # Use 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both
-        # To learn more about changing the languages that are analyzed or customizing the build mode for your analysis,
-        # see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning.
-        # If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how
-        # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages
-    steps:
-    - name: Checkout repository
-      uses: actions/checkout@v4
-
-    # Initializes the CodeQL tools for scanning.
-    - name: Initialize CodeQL
-      uses: github/codeql-action/init@v3
-      with:
-        languages: ${{ matrix.language }}
-        build-mode: ${{ matrix.build-mode }}
-        # If you wish to specify custom queries, you can do so here or in a config file.
-        # By default, queries listed here will override any specified in a config file.
-        # Prefix the list here with "+" to use these queries and those in the config file.
-
-        # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
-        # queries: security-extended,security-and-quality
-
-    # If the analyze step fails for one of the languages you are analyzing with
-    # "We were unable to automatically build your code", modify the matrix above
-    # to set the build mode to "manual" for that language. Then modify this step
-    # to build your code.
-    # ℹ️ Command-line programs to run using the OS shell.
-    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
-    - if: matrix.build-mode == 'manual'
-      shell: bash
-      run: |
-        echo 'If you are using a "manual" build mode for one or more of the' \
-          'languages you are analyzing, replace this with the commands to build' \
-          'your code, for example:'
-        echo '  make bootstrap'
-        echo '  make release'
-        exit 1
-
-    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v3
-      with:
-        category: "/language:${{matrix.language}}"

.github/workflows/daily_checks.yml

@@ -23,10 +23,10 @@ jobs:
   pip-audit:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
       - uses: ./.github/actions/setup-project
       - name: Create requirements.txt
-        run: poetry export --without-hashes --format=requirements.txt > requirements.txt
+        run: pipenv requirements > requirements.txt
       - uses: pypa/gh-action-pip-audit@v1.0.6
         with:
           inputs: requirements.txt
@@ -39,7 +39,7 @@ jobs:
   static-scan:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
       - uses: ./.github/actions/setup-project
       - name: Install bandit
         run: pip install bandit
@@ -69,24 +69,20 @@ jobs:
           # Maps tcp port 5432 on service container to the host
           - 5432:5432
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
       - uses: ./.github/actions/setup-project
       - name: Install application dependencies
         run: make bootstrap
         env:
           SQLALCHEMY_DATABASE_TEST_URI: postgresql://user:password@localhost:5432/test_notification_api
-          NOTIFY_E2E_TEST_EMAIL: ${{ secrets.NOTIFY_E2E_TEST_EMAIL }}
-          NOTIFY_E2E_TEST_HTTP_AUTH_PASSWORD: ${{ secrets.NOTIFY_E2E_TEST_HTTP_AUTH_PASSWORD }}
-          NOTIFY_E2E_TEST_HTTP_AUTH_USER: ${{ secrets.NOTIFY_E2E_TEST_HTTP_AUTH_USER }}
-          NOTIFY_E2E_TEST_PASSWORD: ${{ secrets.NOTIFY_E2E_TEST_PASSWORD }}
       - name: Run server
         run: make run-flask &
         env:
           SQLALCHEMY_DATABASE_TEST_URI: postgresql://user:password@localhost:5432/test_notification_api
-      - name: Run OWASP API Scan
-        uses: zaproxy/action-api-scan@v0.5.0
+      - name: Run OWASP Baseline Scan
+        uses: zaproxy/action-api-scan@v0.1.1
         with:
-          docker_name: 'ghcr.io/zaproxy/zaproxy:weekly'
+          docker_name: 'owasp/zap2docker-weekly'
           target: 'http://localhost:6011/docs/openapi.yml'
           fail_action: true
           allow_issue_writing: false

.github/workflows/deploy-demo.yml

@@ -12,17 +12,17 @@ jobs:
     runs-on: ubuntu-latest
     environment: demo
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v3
       with:
         fetch-depth: 2
 
     - name: Check for changes to Terraform
       id: changed-terraform-files
-      uses: tj-actions/changed-files@v44
+      uses: tj-actions/changed-files@v34
       with:
         files: |
-          terraform/demo/**
-          terraform/shared/**
+          terraform/demo
+          terraform/shared
           .github/workflows/deploy-demo.yml
     - name: Terraform init
       if: steps.changed-terraform-files.outputs.any_changed == 'true'
@@ -45,8 +45,8 @@ jobs:
     - name: Install application dependencies
       run: make bootstrap
 
-    - name: Create requirements.txt
-      run: poetry export --without-hashes --format=requirements.txt > requirements.txt
+    - name: Create requirements.txt because Cloud Foundry does a weird pipenv thing
+      run: pipenv requirements > requirements.txt
 
     - name: Deploy to cloud.gov
       uses: 18f/cg-deploy-action@main
@@ -55,14 +55,10 @@ jobs:
         SECRET_KEY: ${{ secrets.SECRET_KEY }}
         ADMIN_CLIENT_SECRET: ${{ secrets.ADMIN_CLIENT_SECRET }}
         NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY }}
-        NOTIFY_E2E_TEST_EMAIL: ${{ secrets.NOTIFY_E2E_TEST_EMAIL }}
-        NOTIFY_E2E_TEST_PASSWORD: ${{ secrets.NOTIFY_E2E_TEST_PASSWORD }}
-        LOGIN_DOT_GOV_REGISTRATION_URL: "https://secure.login.gov/openid_connect/authorize?acr_values=http%3A%2F%2Fidmanagement.gov%2Fns%2Fassurance%2Fial%2F1&client_id=urn:gov:gsa:openidconnect.profiles:sp:sso:gsa:notify-gov&nonce=NONCE&prompt=select_account&redirect_uri=https://notify-demo.app.cloud.gov/set-up-your-profile&response_type=code&scope=openid+email&state=STATE"
-
       with:
         cf_username: ${{ secrets.CLOUDGOV_USERNAME }}
         cf_password: ${{ secrets.CLOUDGOV_PASSWORD }}
-        cf_org: gsa-tts-benefits-studio
+        cf_org: gsa-tts-benefits-studio-prototyping
         cf_space: notify-demo
         push_arguments: >-
           --vars-file deploy-config/demo.yml
@@ -70,23 +66,10 @@ jobs:
           --var SECRET_KEY="$SECRET_KEY"
           --var ADMIN_CLIENT_SECRET="$ADMIN_CLIENT_SECRET"
           --var NEW_RELIC_LICENSE_KEY="$NEW_RELIC_LICENSE_KEY"
-          --var NOTIFY_E2E_TEST_EMAIL="$NOTIFY_E2E_TEST_EMAIL"
-          --var NOTIFY_E2E_TEST_PASSWORD="$NOTIFY_E2E_TEST_PASSWORD"
-          --var LOGIN_DOT_GOV_REGISTRATION_URL="$LOGIN_DOT_GOV_REGISTRATION_URL"
-
-    - name: Check for changes to templates.json
-      id: changed-templates
-      uses: tj-actions/changed-files@v44
-      with:
-        files: |
-          app/config_files/templates.json
-    - name: Update templates
-      if: steps.changed-templates.outputs.any_changed == 'true'
-      run: cf run-task notify-api-demo --command "flask command update-templates"
 
     - name: Check for changes to egress config
       id: changed-egress-config
-      uses: tj-actions/changed-files@v44
+      uses: tj-actions/changed-files@v34
       with:
         files: |
           deploy-config/egress_proxy/notify-api-demo.*.acl

.github/workflows/deploy-prod.yml

@@ -16,17 +16,17 @@ jobs:
     runs-on: ubuntu-latest
     environment: production
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v3
       with:
         fetch-depth: 2
 
     - name: Check for changes to Terraform
       id: changed-terraform-files
-      uses: tj-actions/changed-files@v44
+      uses: tj-actions/changed-files@v34
       with:
         files: |
-          terraform/production/**
-          terraform/shared/**
+          terraform/production
+          terraform/shared
           .github/workflows/deploy-prod.yml
     - name: Terraform init
       if: steps.changed-terraform-files.outputs.any_changed == 'true'
@@ -49,8 +49,8 @@ jobs:
     - name: Install application dependencies
       run: make bootstrap
 
-    - name: Create requirements.txt
-      run: poetry export --without-hashes --format=requirements.txt > requirements.txt
+    - name: Create requirements.txt because Cloud Foundry does a weird pipenv thing
+      run: pipenv requirements > requirements.txt
 
     - name: Deploy to cloud.gov
       uses: 18f/cg-deploy-action@main
@@ -59,14 +59,10 @@ jobs:
         SECRET_KEY: ${{ secrets.SECRET_KEY }}
         ADMIN_CLIENT_SECRET: ${{ secrets.ADMIN_CLIENT_SECRET }}
         NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY }}
-        NOTIFY_E2E_TEST_EMAIL: ${{ secrets.NOTIFY_E2E_TEST_EMAIL }}
-        NOTIFY_E2E_TEST_PASSWORD: ${{ secrets.NOTIFY_E2E_TEST_PASSWORD }}
-        LOGIN_DOT_GOV_REGISTRATION_URL: "https://secure.login.gov/openid_connect/authorize?acr_values=http%3A%2F%2Fidmanagement.gov%2Fns%2Fassurance%2Fial%2F1&client_id=urn:gov:gsa:openidconnect.profiles:sp:sso:gsa:notify-gov&nonce=NONCE&prompt=select_account&redirect_uri=https://beta.notify.gov/set-up-your-profile&response_type=code&scope=openid+email&state=STATE"
-
       with:
         cf_username: ${{ secrets.CLOUDGOV_USERNAME }}
         cf_password: ${{ secrets.CLOUDGOV_PASSWORD }}
-        cf_org: gsa-tts-benefits-studio
+        cf_org: gsa-tts-benefits-studio-prototyping
         cf_space: notify-production
         push_arguments: >-
           --vars-file deploy-config/production.yml
@@ -74,23 +70,10 @@ jobs:
           --var SECRET_KEY="$SECRET_KEY"
           --var ADMIN_CLIENT_SECRET="$ADMIN_CLIENT_SECRET"
           --var NEW_RELIC_LICENSE_KEY="$NEW_RELIC_LICENSE_KEY"
-          --var NOTIFY_E2E_TEST_EMAIL="$NOTIFY_E2E_TEST_EMAIL"
-          --var NOTIFY_E2E_TEST_PASSWORD="$NOTIFY_E2E_TEST_PASSWORD"
-          --var LOGIN_DOT_GOV_REGISTRATION_URL="$LOGIN_DOT_GOV_REGISTRATION_URL"
-
-    - name: Check for changes to templates.json
-      id: changed-templates
-      uses: tj-actions/changed-files@v44
-      with:
-        files: |
-          app/config_files/templates.json
-    - name: Update templates
-      if: steps.changed-templates.outputs.any_changed == 'true'
-      run: cf run-task notify-api-production --command "flask command update-templates"
 
     - name: Check for changes to egress config
       id: changed-egress-config
-      uses: tj-actions/changed-files@v44
+      uses: tj-actions/changed-files@v34
       with:
         files: |
           deploy-config/egress_proxy/notify-api-production.*.acl

.github/workflows/deploy.yml

@@ -17,17 +17,17 @@ jobs:
 
     environment: staging
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v3
       with:
         fetch-depth: 2
 
     - name: Check for changes to Terraform
       id: changed-terraform-files
-      uses: tj-actions/changed-files@v44
+      uses: tj-actions/changed-files@v34
       with:
         files: |
-          terraform/staging/**
-          terraform/shared/**
+          terraform/staging
+          terraform/shared
           .github/workflows/deploy.yml
     - name: Terraform init
       if: steps.changed-terraform-files.outputs.any_changed == 'true'
@@ -50,8 +50,8 @@ jobs:
     - name: Install application dependencies
       run: make bootstrap
 
-    - name: Create requirements.txt
-      run: poetry export --without-hashes --format=requirements.txt > requirements.txt
+    - name: Create requirements.txt because Cloud Foundry does a weird pipenv thing
+      run: pipenv requirements > requirements.txt
 
     - name: Deploy to cloud.gov
       uses: 18f/cg-deploy-action@main
@@ -60,14 +60,10 @@ jobs:
         SECRET_KEY: ${{ secrets.SECRET_KEY }}
         ADMIN_CLIENT_SECRET: ${{ secrets.ADMIN_CLIENT_SECRET }}
         NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY }}
-        NOTIFY_E2E_TEST_EMAIL: ${{ secrets.NOTIFY_E2E_TEST_EMAIL }}
-        NOTIFY_E2E_TEST_PASSWORD: ${{ secrets.NOTIFY_E2E_TEST_PASSWORD }}
-        LOGIN_DOT_GOV_REGISTRATION_URL: "https://secure.login.gov/openid_connect/authorize?acr_values=http%3A%2F%2Fidmanagement.gov%2Fns%2Fassurance%2Fial%2F1&client_id=urn:gov:gsa:openidconnect.profiles:sp:sso:gsa:notify-gov&nonce=NONCE&prompt=select_account&redirect_uri=https://notify-staging.app.cloud.gov/set-up-your-profile&response_type=code&scope=openid+email&state=STATE"
-
       with:
         cf_username: ${{ secrets.CLOUDGOV_USERNAME }}
         cf_password: ${{ secrets.CLOUDGOV_PASSWORD }}
-        cf_org: gsa-tts-benefits-studio
+        cf_org: gsa-tts-benefits-studio-prototyping
         cf_space: notify-staging
         push_arguments: >-
           --vars-file deploy-config/staging.yml
@@ -75,23 +71,10 @@ jobs:
           --var SECRET_KEY="$SECRET_KEY"
           --var ADMIN_CLIENT_SECRET="$ADMIN_CLIENT_SECRET"
           --var NEW_RELIC_LICENSE_KEY="$NEW_RELIC_LICENSE_KEY"
-          --var NOTIFY_E2E_TEST_EMAIL="$NOTIFY_E2E_TEST_EMAIL"
-          --var NOTIFY_E2E_TEST_PASSWORD="$NOTIFY_E2E_TEST_PASSWORD"
-          --var LOGIN_DOT_GOV_REGISTRATION_URL="$LOGIN_DOT_GOV_REGISTRATION_URL"
-
-    - name: Check for changes to templates.json
-      id: changed-templates
-      uses: tj-actions/changed-files@v44
-      with:
-        files: |
-          app/config_files/templates.json
-    - name: Update templates
-      if: steps.changed-templates.outputs.any_changed == 'true'
-      run: cf run-task notify-api-staging --command "flask command update-templates"
 
     - name: Check for changes to egress config
       id: changed-egress-config
-      uses: tj-actions/changed-files@v44
+      uses: tj-actions/changed-files@v34
       with:
         files: |
           deploy-config/egress_proxy/notify-api-staging.*.acl

.github/workflows/drift.yml

@@ -13,7 +13,7 @@ jobs:
     environment: staging
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
 
       - name: Check for drift
         uses: dflook/terraform-check@v1
@@ -31,7 +31,7 @@ jobs:
     environment: demo
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
         with:
           ref: 'production'
 
@@ -51,7 +51,7 @@ jobs:
     environment: production
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
         with:
           ref: 'production'
 

.github/workflows/restage-apps.yml

@@ -23,7 +23,7 @@ jobs:
         with:
           cf_username: ${{ secrets.CLOUDGOV_USERNAME }}
           cf_password: ${{ secrets.CLOUDGOV_PASSWORD }}
-          cf_org: gsa-tts-benefits-studio
+          cf_org: gsa-tts-benefits-studio-prototyping
           cf_space: notify-${{ inputs.environment }}
           full_command: "cf restage --strategy rolling notify-${{matrix.app}}-${{inputs.environment}}"
       - name: Restage ${{matrix.app}} egress
@@ -31,6 +31,6 @@ jobs:
         with:
           cf_username: ${{ secrets.CLOUDGOV_USERNAME }}
           cf_password: ${{ secrets.CLOUDGOV_PASSWORD }}
-          cf_org: gsa-tts-benefits-studio
+          cf_org: gsa-tts-benefits-studio-prototyping
           cf_space: notify-${{ inputs.environment }}-egress
           full_command: "cf restage --strategy rolling egress-proxy-notify-${{matrix.app}}-${{inputs.environment}}"

.github/workflows/terraform-demo.yml

@@ -16,7 +16,7 @@ jobs:
     environment: demo
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v2
 
       - name: Terraform format
         id: format

.github/workflows/terraform-production.yml

@@ -16,7 +16,7 @@ jobs:
     environment: production
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v2
 
       - name: Terraform format
         id: format

.github/workflows/terraform-staging.yml

@@ -16,7 +16,7 @@ jobs:
     environment: staging
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v2
 
       - name: Terraform format
         id: format

.gitignore

@@ -92,7 +92,3 @@ varsfile*
 secrets.auto.tfvars
 terraform.tfstate
 terraform.tfstate.backup
-
-
-# Pyenv
-.python-version

.pre-commit-config.yaml

@@ -1,24 +0,0 @@
-# See https://pre-commit.com for more information
-# See https://pre-commit.com/hooks.html for more hooks
-repos:
--   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.6.0
-    hooks:
-    -   id: trailing-whitespace
-    -   id: end-of-file-fixer
-    -   id: check-yaml
-    -   id: check-added-large-files
-    -   id: debug-statements
-    -   id: check-merge-conflict
-    -   id: check-toml
-    -   id: check-ast
-    -   id: fix-byte-order-marker
-    -   id: detect-aws-credentials
-        args: [--allow-missing-credentials]
-    -   id: detect-private-key
-    -   id: mixed-line-ending
--   repo: https://github.com/Yelp/detect-secrets
-    rev: v1.5.0
-    hooks:
-    -   id: detect-secrets
-        args: ['--baseline', '.ds.baseline']

.profile

@@ -3,7 +3,5 @@
 # https://docs.cloudfoundry.org/devguide/deploy-apps/deploy-app.html#profile
 ##
 
-export http_proxy=$egress_proxy
 export https_proxy=$egress_proxy
 export NEW_RELIC_PROXY_HOST=$egress_proxy
-export no_proxy="apps.internal"

.python-version

@@ -0,0 +1 @@
+3.9

CONTRIBUTING.md

@@ -16,4 +16,4 @@ As part of a U.S. government agency, the General Services Administration (GSA)
 
 This project is in the public domain within the United States, and applicable copyright and related rights in the work worldwide are waived through the [CC0 1.0 Universal public domain dedication](https://creativecommons.org/publicdomain/zero/1.0/).
 
-All contributions to this project will be released under the CC0 dedication. By submitting a pull request or issue, you are agreeing to comply with this waiver of copyright interest.
+All contributions to this project will be released under the CC0 dedication. By submitting a pull request or issue, you are agreeing to comply with this waiver of copyright interest.

LICENSE.md

@@ -50,4 +50,4 @@ As stated in [CONTRIBUTING](CONTRIBUTING.md), all contributions to this project
 
 ## Trademark
 
-GSA owns, reserves, and retains all rights, title, and interest in all trademarks owned by GSA. The User does not acquire any right or interest of any kind in any GSA trademark because of their use of the Modified Version or any of the modifications made to the Work by GSA.
+GSA owns, reserves, and retains all rights, title, and interest in all trademarks owned by GSA. The User does not acquire any right or interest of any kind in any GSA trademark because of their use of the Modified Version or any of the modifications made to the Work by GSA.

Makefile

@@ -12,14 +12,9 @@ GIT_COMMIT ?= $(shell git rev-parse HEAD)
 .PHONY: bootstrap
 bootstrap: ## Set up everything to run the app
 	make generate-version-file
-	poetry self update
-	poetry self add poetry-dotenv-plugin
-	poetry lock --no-update
-	poetry install --sync --no-root
-	poetry run pre-commit install
+	pipenv install --dev
 	createdb notification_api || true
-	createdb test_notification_api || true
-	(poetry run flask db upgrade) || true
+	(pipenv run flask db upgrade) || true
 
 .PHONY: bootstrap-with-docker
 bootstrap-with-docker: ## Build the image to run the app in Docker
@@ -27,45 +22,30 @@ bootstrap-with-docker: ## Build the image to run the app in Docker
 
 .PHONY: run-procfile
 run-procfile:
-	poetry run honcho start -f Procfile.dev
-
-.PHONY: avg-complexity
-avg-complexity:
-	echo "*** Shows average complexity in radon of all code ***"
-	poetry run radon cc ./app -a -na
-
-.PHONY: too-complex
-too-complex:
-	echo "*** Shows code that got a rating of C, D or F in radon ***"
-	poetry run radon cc ./app -a -nc
+	pipenv run honcho start -f Procfile.dev
 
 .PHONY: run-flask
 run-flask: ## Run flask
-	poetry run newrelic-admin run-program flask run -p 6011 --host=0.0.0.0
+	pipenv run newrelic-admin run-program flask run -p 6011 --host=0.0.0.0
 
 .PHONY: run-celery
 run-celery: ## Run celery, TODO remove purge for staging/prod
-	poetry run celery -A run_celery.notify_celery purge -f
-	poetry run newrelic-admin run-program celery \
+	pipenv run celery -A run_celery.notify_celery purge -f
+	pipenv run newrelic-admin run-program celery \
 		-A run_celery.notify_celery worker \
 		--pidfile="/tmp/celery.pid" \
 		--loglevel=INFO \
 		--concurrency=4
 
-
-.PHONY: dead-code
-dead-code:
-	poetry run vulture ./app --min-confidence=100
-
 .PHONY: run-celery-beat
 run-celery-beat: ## Run celery beat
-	poetry run celery \
+	pipenv run celery \
 	-A run_celery.notify_celery beat \
 	--loglevel=INFO
 
 .PHONY: cloudgov-user-report
 cloudgov-user-report:
-	@poetry run python -m terraform.ops.cloudgov_user_report
+	@pipenv run python -m terraform.ops.cloudgov_user_report
 
 .PHONY: help
 help:
@@ -78,33 +58,27 @@ generate-version-file: ## Generates the app version file
 .PHONY: test
 test: export NEW_RELIC_ENVIRONMENT=test
 test: ## Run tests and create coverage report
-	poetry run black .
-	poetry run flake8 .
-	poetry run isort --check-only ./app ./tests
-	poetry run coverage run --omit=*/notifications_utils/*,*/migrations/* -m pytest --maxfail=10
-
-	poetry run coverage report -m --fail-under=95
-	poetry run coverage html -d .coverage_cache
-
-.PHONY: py-lock
-py-lock: ## Syncs dependencies and updates lock file without performing recursive internal updates
-	poetry lock --no-update
-	poetry install --sync
+	pipenv run flake8 .
+	pipenv run isort --check-only ./app ./tests
+	pipenv run coverage run --omit=*/notifications_utils/* -m pytest --maxfail=10
+	pipenv run coverage report --fail-under=50
+	pipenv run coverage html -d .coverage_cache
 
 .PHONY: freeze-requirements
 freeze-requirements: ## Pin all requirements including sub dependencies into requirements.txt
-	poetry export --without-hashes --format=requirements.txt > requirements.txt
+	pipenv lock
+	pipenv requirements
 
 .PHONY: audit
 audit:
-	poetry requirements > requirements.txt
-	poetry requirements --dev > requirements_for_test.txt
-	poetry run pip-audit -r requirements.txt
-	poetry run pip-audit -r requirements_for_test.txt
+	pipenv requirements > requirements.txt
+	pipenv requirements --dev > requirements_for_test.txt
+	pipenv run pip-audit -r requirements.txt
+	-pipenv run pip-audit -r requirements_for_test.txt
 
 .PHONY: static-scan
 static-scan:
-	poetry run bandit -r app/
+	pipenv run bandit -r app/
 
 .PHONY: clean
 clean:

Pipfile

@@ -0,0 +1,84 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+alembic = "==1.7.7"
+amqp = "==5.1.1"
+arrow = "==1.2.2"
+asn1crypto = "==1.5.1"
+async-timeout = "==4.0.2"
+attrs = "==21.4.0"
+awscli = "==1.24.8"
+bcrypt = "==3.2.2"
+beautifulsoup4 = "==4.12.2"
+billiard = "==3.6.4.0"
+bleach = "==4.1.0"
+blinker = "~=1.4"
+boto3 = "==1.23.8"
+botocore = "==1.26.8"
+cachetools = "==5.1.0"
+celery = {version = "==5.2.7", extras = ["redis"]}
+certifi = ">=2022.12.7"
+cffi = "==1.15.0"
+charset-normalizer = "==2.0.12"
+click = "==8.1.3"
+click-datetime = "==0.2"
+click-didyoumean = "==0.3.0"
+click-plugins = "==1.1.1"
+click-repl = "==0.2.0"
+colorama = "==0.4.4"
+defusedxml = "==0.7.1"
+deprecated = "==1.2.13"
+dnspython = "==2.2.1"
+docopt = "==0.6.2"
+docutils = "==0.16"
+eventlet = "==0.33.1"
+flask = "~=2.3"
+flask-bcrypt = "==1.0.1"
+flask-marshmallow = "==0.14.0"
+flask-migrate = "==3.1.0"
+flask-redis = "==0.4.0"
+flask-sqlalchemy = {version = "==2.5.1", ref = "aa7a61a5357cf6f5dcc135d98c781192457aa6fa", git = "https://github.com/pallets-eco/flask-sqlalchemy.git"}
+gunicorn = {version = "==20.1.0", extras = ["eventlet"], ref = "1299ea9e967a61ae2edebe191082fd169b864c64", git = "https://github.com/benoitc/gunicorn.git"}
+iso8601 = "==1.0.2"
+itsdangerous = "==2.1.2"
+jsonschema = {version = "==4.17.3", extras = ["format"]}
+lxml = "==4.9.1"
+marshmallow = "==3.19.0"
+marshmallow-sqlalchemy = "==0.29.0"
+notifications-python-client = "==6.3.0"
+oscrypto = "==1.3.0"
+psycopg2-binary = "==2.9.3"
+pyjwt = "==2.7.0"
+python-dotenv = "==1.0.0"
+sqlalchemy = "==1.4.40"
+werkzeug = "~=2.3"
+# gds metrics packages
+prometheus-client = "==0.14.1"
+gds-metrics = {version = "==0.2.4", ref = "6f1840a57b6fb1ee40b7e84f2f18ec229de8aa72", git = "https://github.com/alphagov/gds_metrics_python.git"}
+packaging = "==23.1"
+notifications-utils = {editable = true, ref = "main", git = "https://github.com/GSA/notifications-utils.git"}
+newrelic = "*"
+
+[dev-packages]
+flake8 = "==4.0.1"
+flake8-bugbear = "==22.4.25"
+isort = "==5.10.1"
+moto = "==3.1.9"
+pytest = "==7.1.2"
+pytest-env = "==0.6.2"
+pytest-mock = "==3.11.1"
+pytest-cov = "==3.0.0"
+pytest-xdist = "==2.5.0"
+freezegun = "==1.2.1"
+requests-mock = "==1.9.3"
+jinja2-cli = {version = "==0.8.2", extras = ["yaml"]}
+pip-audit = "*"
+bandit = "*"
+honcho = "*"
+cloudfoundry-client = "*"
+
+[requires]
+python_version = "3.9"

README.md

@@ -1,530 +1,132 @@
-![notify-logo](https://github.com/GSA/notifications-api/assets/4156602/6b2905d2-a232-4414-8815-25dba6008f17)
+# US Notify API
 
-# Notify.gov API
-
-This project is the core of [Notify.gov](https://notify-demo.app.cloud.gov).
-It's cloned from the brilliant work of the team at
-[GOV.UK Notify](https://github.com/alphagov/notifications-api), cheers!
+This project is the core of [Notify](https://notifications-admin.app.cloud.gov/). It's cloned from the brilliant work of the team at [GOV.UK Notify](https://github.com/alphagov/notifications-api), cheers!
 
 This repo contains:
 
-- A public-facing REST API for Notify.gov, which teams can integrate with using
- [API clients built by UK](https://www.notifications.service.gov.uk/documentation).
-- An internal-only REST API built using Flask to manage services, users,
-  templates, etc., which the
-  [Notify.gov Admin UI](http://github.com/18F/notifications-admin) talks to.
-- Asynchronous workers built using Celery to put things on queues and read them
-  off to be processed, sent to providers, updated, etc.
+- A public-facing REST API for Notify, which teams can integrate with using [API clients built by UK](https://www.notifications.service.gov.uk/documentation)
+- An internal-only REST API built using Flask to manage services, users, templates, etc., which the [admin UI](http://github.com/18F/notifications-admin) talks to)
+- Asynchronous workers built using Celery to put things on queues and read them off to be processed, sent to providers, updated, etc.
 
 Our other repositories are:
 
 - [notifications-admin](https://github.com/GSA/notifications-admin)
+- [notifications-utils](https://github.com/GSA/notifications-utils)
 - [us-notify-compliance](https://github.com/GSA/us-notify-compliance/)
 - [notify-python-demo](https://github.com/GSA/notify-python-demo)
 
-## Before You Start
+## Documentation, here and elsewhere
 
-You will need the following items:
-
-- An active cloud.gov account with the correct permissions - speak with your
-  onboarding buddy for help with
-  [setting up an account](https://cloud.gov/sign-up/) (requires a `.mil`,
-  `.gov`, or `.fed.us` email address) and getting access to the
-  `notify-local-dev` and `notify-staging` spaces.
-- Admin priviliges and SSH access on your machine; you may need to work with
-  your organization's IT support staff if you're not sure or don't currently
-  have this access.
+### About Notify
 
-## Local Environment Setup
+- [Roadmap](https://notifications-admin.app.cloud.gov/features/roadmap)
+- [Using the API](./docs/api-usage.md)
+- [Architectural Decision Records](./docs/adrs/)
 
-This project currently works with these major versions of the following main
-components:
+### Infrastructure
 
-- Python 3.12.x
-- PostgreSQL 15.x (version 12.x is used in the hosted environments)
+- [Overview, setup, and onboarding](./docs/infra-overview.md)
+- [Database management](./docs/database-management.md)
+- [Celery queues and tasks](./docs/queues-and-tasks.md)
 
-These instructions will walk you through how to set your machine up with all of
-the required tools for this project.
+### Common dev work
 
-### Project Pre-Requisite Setup
+- [Local setup](#local-setup)
+- [Testing](./docs/testing.md), both automated and manual
+- [Deploying](./docs/deploying.md)
+- [Running one-off tasks](./docs/one-off-tasks.md)
 
-On MacOS, using [Homebrew](https://brew.sh/) for package management is highly
-recommended. This helps avoid some known installation issues. Start by following
-the installation instructions on the Homebrew homepage.
+## UK docs that may still be helpful
 
-**Note:** You will also need Xcode or the Xcode Command Line Tools installed. The
-quickest way to do this is is by installing the command line tools in the shell:
+- [Writing public APIs](docs/writing-public-apis.md)
+- [Updating dependencies](https://github.com/alphagov/notifications-manuals/wiki/Dependencies)
 
-```sh
-xcode-select –-install
-```
+## Local setup
 
-#### Homebrew Setup
+### Common steps
 
-If this is your first time installing Homebrew on your machine, you may need to
-add its binaries to your system's `$PATH` environment variable so that you can
-use the `brew` command. Try running `brew help` to see if Homebrew is
-recognized and runs properly. If that fails, then you'll need to add a
-configuration line to wherever your `$PATH` environment variable is set.
+On MacOS, using [Homebrew](https://brew.sh/) for package management is highly recommended. This helps avoid some known installation issues.
 
-Your system `$PATH` environment variable is likely set in one of these
-locations:
+1. Install pre-requisites for setup:
+    * [jq](https://stedolan.github.io/jq/): `brew install jq`
+    * [terraform](https://www.terraform.io/): `brew install terraform` or `brew install tfenv` and use `tfenv` to install `terraform ~> 1.4.0`
+    * [cf-cli@8](https://docs.cloudfoundry.org/cf-cli/install-go-cli.html): `brew install cloudfoundry/tap/cf-cli@8`
+    * [postgresql](https://www.postgresql.org/): `brew install postgresql@15` (Homebrew requires a version pin, but any recent version will work)
+    * [redis](https://redis.io/): `brew install redis`
+    * [pyenv](https://github.com/pyenv/pyenv): `brew install pyenv`
+1. [Log into cloud.gov](https://cloud.gov/docs/getting-started/setup/#set-up-the-command-line): `cf login -a api.fr.cloud.gov --sso`
+1. Ensure you have access to the `notify-local-dev` and `notify-staging` spaces in cloud.gov
+1. Run the development terraform with:
 
-For BASH shells:
-- `~/.bashrc`
-- `~/.bash_profile`
-- `~/.profile`
+        ```
+        $ cd terraform/development
+        $ ./run.sh
+        ```
 
-For ZSH shells:
-- `~/.zshrc`
-- `~/.zprofile`
+1. If you want to send data to New Relic from your local develpment environment, set `NEW_RELIC_LICENSE_KEY` within `.env`
+1. Follow the instructions for either `Direct installation` or `Docker installation` below
 
-There may be different files that you need to modify for other shell
-environments.
+### Direct installation
 
-Which file you need to modify depends on whether or not you are running an
-interactive shell or a login shell
-(see [this Stack Overflow post](https://stackoverflow.com/questions/18186929/what-are-the-differences-between-a-login-shell-and-interactive-shell)
-for an explanation of the differences).  If you're still not sure, please ask
-the team for help!
+1. Set up Postgres && Redis on your machine
 
-Once you determine which file you'll need to modify, add these lines before any
-lines that add or modify the `$PATH` environment variable; near or at the top
-of the file is appropriate:
+1. Install [pipenv](https://pipenv.pypa.io/en/latest/)
 
-```sh
-# Homebrew setup
-eval "$(/opt/homebrew/bin/brew shellenv)"
-```
+1. Run the project setup
 
-This will make sure Homebrew gets setup correctly. Once you make these changes,
-either start a new shell session or source the file
-(`source ~/.FILE-YOU-MODIFIED`) you modified to have your system recognize the
-changes.
+    `make bootstrap`
 
-Verify that Homebrew is now working by trying to run `brew help` again.
+1. Run the web server and background worker
 
-### System-Level Package Installation
+    `make run-procfile`
 
-There are several packages you will need to install for your system in order to
-get the app running (and these are good to have in general for any software
-development).
+1. Or run them individually:
 
-Start off with these packages since they're quick and don't require additional
-configuration after installation to get working out of the box:
+    * Run Flask (web server)
 
-- [jq](https://stedolan.github.io/jq/) - for working with JSON in the command
-  line
-- [git](https://git-scm.com/) - for version control management
-- [tfenv](https://github.com/tfutils/tfenv) - for managing
-  [Terraform](https://www.terraform.io/) installations
-- [cf-cli@8](https://docs.cloudfoundry.org/cf-cli/install-go-cli.html) - for
-  working with a Cloud Foundry platform (e.g., cloud.gov)
-- [redis](https://redis.io/) - required as the backend for the API's
-  asynchronous job processing
-- [vim](https://www.vim.org/) - for editing files more easily in the command
-  line
-- [wget](https://www.gnu.org/software/wget/) - for retrieving files in the
-  command line
+        `make run-flask`
 
-You can install them by running the following:
+    * Run Celery (background worker)
 
-```sh
-brew install jq git tfenv cloudfoundry/tap/cf-cli@8 redis vim wget
-```
+        `make run-celery`
 
-#### Terraform Installation
 
-As a part of the installation above, you just installed `tfenv` to manage
-Terraform installations. This is great, but you still need to install Terraform
-itself, which can be done with this command:
+### VS Code && Docker installation
 
-```sh
-tfenv install "latest:^1.7"
-tfenv use 1.7.x # x = the patch version installed
-```
+If you're working in VS Code, you can also leverage Docker for a containerized dev environment
 
-#### Python Installation
+1. Uncomment the `Local Docker setup` lines in `.env` and comment out the `Local direct setup` lines.
 
-Now we're going to install a tool to help us manage Python versions and
-virtual environments on our system.  First, we'll install
-[pyenv](https://github.com/pyenv/pyenv) and one of its plugins,
-[pyenv-virtualenv](https://github.com/pyenv/pyenv-virtualenv), with Homebrew:
+1. Install the Remote-Containers plug-in in VS Code
 
-```sh
-brew install pyenv pyenv-virtualenv
-```
+1. With Docker running, create the network:
 
-When these finish installing, you'll need to make another adjustment in the
-file that you adjusted for your `$PATH` environment variable and Homebrew's
-setup. Open the file, and add these lines to it:
+    `docker network create notify-network`
 
-```
-# pyenv setup
-export PYENV_ROOT="$HOME/.pyenv"
-command -v pyenv >/dev/null || export PATH="$PYENV_ROOT/bin:$PATH"
-eval "$(pyenv init -)"
-eval "$(pyenv virtualenv-init -)"
-```
+1. Using the command palette (shift+cmd+p) or green button thingy in the bottom left, search and select “Remote Containers: Open Folder in Container...” When prompted, choose **devcontainer-api** folder (note: this is a *subfolder* of notifications-api). This will start the container in a new window, replacing the current one.
 
-Once again, start a new shell session or source the file in your current shell
-session to make the changes take effect.
+1. Wait a few minutes while things happen 🍵
 
-Now we're ready to install the Python version we need with `pyenv`, like so:
+1. Open a VS Code terminal and run the Flask application:
 
-```sh
-pyenv install 3.12
-```
+    `make run-flask`
 
-This will install the latest version of Python 3.12.
+1. Open another VS Code terminal and run Celery:
 
-_NOTE: This project currently runs on Python 3.12.x._
+    `make run-celery`
 
-#### Python Dependency Installation
+NOTE: when you change .env in the future, you'll need to rebuild the devcontainer for the change to take effect. VS Code _should_ detect the change and prompt you with a toast notification during a cached build. If not, you can find a manual rebuild in command pallette or just `docker rm` the notifications-api container.
 
-Lastly, we need to install the tool we use to manage Python dependencies within
-the project, which is [poetry](https://python-poetry.org/).
+### Known installation issues
 
-Visit the
-[official installer instructions page](https://python-poetry.org/docs/#installing-with-the-official-installer)
-and follow the steps to install Poetry directly with the script.
+On M1 Macs, if you get a `fatal error: 'Python.h' file not found` message, try a different method of installing Python. Installation via `pyenv` is known to work.
 
-This will ensure `poetry` doesn't conflict with any project virtual environments
-and can update itself properly.
-
-#### PostgreSQL installation
-
-We now need to install a database - this project uses PostgreSQL, and Homebrew
-requires a version number to be included with it when installing it:
-
-```sh
-brew install postgresql@15
-```
-
-_NOTE: This project currently works with PostgreSQL version 15.x; version 12.x is currently used in our hosted environments._
-
-_NOTE: If you have a pre-existing instance of PSQL installed because of another product like PGAdmin, your database configuration may differ from the instructions above, which uses Homebrew to install and configure PostgreSQL.  If this is the case for you, you may have to either account for slightly different user permissions with the database, or uninstall PGAdmin and/or PostgreSQL itself, and reinstall it with Homebrew to follow the steps above._
-
-You'll now need to modify (or create, if it doesn't already exist) the `$PATH`
-environment variable to include the PostgreSQL binaries. Open the file you have
-worked with before to adjust your shell environment with the previous steps and
-do one of the following:
-
-If you already have a line that modifies the `$PATH` environment variable, just
-add this path into the existing string:
-
-```
-/opt/homebrew/opt/postgresql@15/bin
-```
-
-If you don't have a line for your `$PATH` environment variable, add it in like
-this, which will include the PostgreSQL binaries:
-
-```
-export PATH="/opt/homebrew/opt/postgresql@15/bin:$PATH
-```
-
-_NOTE: You don't want to overwrite your existing `$PATH` environment variable! Hence the reason why it is included on the end like this; paths are separated by a colon._
-
-#### Starting PostgreSQL and Redis
-
-With both PostgreSQL and Redis installed, you now need to start the services.
-Run this command so that they're available at all times going forward on your
-machine:
-
-```sh
-brew services start postgresql@15
-brew services start redis
-```
-
-If they're already running, you can run this command instead to make sure the
-latest updates are applied to both services:
-
-```sh
-brew services restart postgresql@15
-brew services restart redis
-```
-
-### First-Time Project Setup
-
-Once all of pre-requisites for the project are installed and you have a
-cloud.gov account, you can now set up the API project and get things running
-locally!
-
-First, clone the repository in the directory of your choosing on your machine:
-
-```sh
-git clone git@github.com:GSA/notifications-api.git
-```
-
-Now go into the project directory (`notifications-api` by default), create a
-virtual environment, and set the local Python version to point to the virtual
-environment (assumes version Python `3.12.2` is what is installed on your
-machine):
-
-```sh
-cd notifications-api
-pyenv virtualenv 3.12.2 notify-api
-pyenv local notify-api
-```
-
-_If you're not sure which version of Python was installed with `pyenv`, you can check by running `pyenv versions` and it'll list everything available currently._
-
-Now [log into cloud.gov](https://cloud.gov/docs/getting-started/setup/#set-up-the-command-line)
-in the command line by using this command:
-
-```sh
-cf login -a api.fr.cloud.gov --sso
-```
-If you are offered a choice of orgs, select `gsa-tts-benefits-studio`.
-For the space, choose `notify-local-dev` to start with (assuming you are
-setting up local development).
-
-_REMINDER: Ensure you have access to the `notify-local-dev` and `notify-staging` spaces in cloud.gov_
-
-Now run the development Terraform setup by navigating to the development
-folder and running the script in it:
-
-```sh
-cd terraform/development
-./run.sh
-```
-
-If this runs correctly, Terraform will ask you if you want to create some
-resources. Answer `yes`.
-
-The script will also create a local `.env` file for you in the project's
-root directory, which will include a handful of project-specific environment
-variables.
-
-Lastly, if you didn't already start PostgreSQL and Redis above, be sure to do
-so now:
-
-```sh
-brew services start postgresql@15
-brew services start redis
-```
-
-#### Upgrading Python in existing projects
-
-If you're upgrading an existing project to a newer version of Python, you can
-follow these steps to get yourself up-to-date.
-
-First, use `pyenv` to install the newer version of Python you'd like to use;
-we'll use `3.12` in our example here since we recently upgraded to this version:
-
-```sh
-pyenv install 3.12
-```
-
-Next, delete the virtual environment you previously had set up.  If you followed
-the instructions above with the first-time set up, you can do this with `pyenv`:
-
-```sh
-pyenv virtualenv-delete notify-api
-```
-
-Now, make sure you are in your project directory and recreate the same virtual
-environment with the newer version of Python you just installed:
-
-```sh
-cd notifications-api
-pyenv virtualenv 3.12.2 notify-api
-pyenv local notify-api
-```
-
-At this point, proceed with the rest of the instructions here in the README and
-you'll be set with an upgraded version of Python.
-
-_If you're not sure about the details of your current virtual environment, you can run `poetry env info` to get more information. If you've been using `pyenv` for everything, you can also see all available virtual environments with `pyenv virtualenvs`._
-
-
-### Final environment setup
-
-There's one final thing to adjust in the newly created `.env` file.  This
-project has support for end-to-end (E2E) tests and has some additional checks
-for the presence of an E2E test user so that it can be authenticated properly.
-
-In the `.env` file, you should see this section:
-
-```
-#############################################################
-
-# E2E Testing
-
-NOTIFY_E2E_TEST_EMAIL=example@fake.gov
-NOTIFY_E2E_TEST_PASSWORD="don't write secrets to the sample file"
-```
-
-You can leave the email address alone or change it to something else to your
-liking.
-
-**You should absolutely change the `NOTIFY_E2E_TEST_PASSWORD` environment
-variable to something else, preferably a lengthy passphrase.**
-
-With those two environment variable set, the database migrations will run
-properly and an E2E test user will be ready to go for use in the admin project.
-
-_Note:  Whatever you set these two environment variables to, you'll need to
-match their values on the admin side.  Please see the admin README and
-documentation for more details._
-
-## Running the Project and Routine Maintenance
-
-The first time you run the project you'll need to run the project setup from the
-root project directory:
-
-```sh
-make bootstrap
-```
-
-This command is handled by the `Makefile` file in the root project directory, as
-are a few others.
-
-_NOTE: You'll want to occasionally run `make bootstrap` to keep your project up-to-date, especially when there are dependency updates._
-
-Now you can run the web server and background workers for asynchronous jobs:
-
-```sh
-make run-procfile
-```
-
-If it runs correctly, you will be able to visit http://127.0.0.1:6011/ and see
-JSON from the API in your web browser.
-
-This will run all of the services within the same shell session.  If you need to
-run them separately to help with debugging or tracing logs, you can do so by
-opening three sepearate shell sessions and running one of these commands in each
-one separately:
-
-- `make run-celery` - Handles the asynchronous jobs
-- `make run-celery-beat` - Handles the scheduling of asynchronous jobs
-- `make run-flask` - Runs the web server
-
-## Python Dependency Management
-
-We're using [`Poetry`](https://python-poetry.org/) for managing our Python
-dependencies and local virtual environments. When it comes to managing the
-Python dependencies, there are a couple of things to bear in mind.
-
-For situations where you manually manipulate the `pyproject.toml` file, you
-should use the `make py-lock` command to sync the `poetry.lock` file. This will
-ensure that you don't inadvertently bring in other transitive dependency updates
-that have not been fully tested with the project yet.
-
-If you're just trying to update a dependency to a newer (or the latest) version,
-you should let Poetry take care of that for you by running the following:
-
-```sh
-poetry update <dependency> [<dependency>...]
-```
-
-You can specify more than one dependency together. With this command, Poetry
-will do the following for you:
-
-- Find the latest compatible version(s) of the specified dependency/dependencies
-- Install the new versions
-- Update and sync the `poetry.lock` file
-
-In either situation, once you are finished and have verified the dependency
-changes are working, please be sure to commit both the `pyproject.toml` and
-`poetry.lock` files.
-
-## Known Installation Issues
-
-### Python Installation Errors
-
-On M1 Macs, if you get a `fatal error: 'Python.h' file not found` message, try a
-different method of installing Python. The recommended approach is to use
-[`pyenv`](https://github.com/pyenv/pyenv), as noted above in the installation
-instructions.
-
-If you're using PyCharm for Python development, we've noticed some quirkiness
-with the IDE and the interaction between Poetry and virtual environment
-management that could cause a variety of problems to come up during project
-setup and dependency management. Other tools, such as Visual Studio Code, have
-proven to be a smoother experience for folks.
-
-### PostgreSQL Installation Errors
-
-A direct installation of PostgreSQL will not put the `createdb` command on your
-`$PATH`. It can be added there in your shell startup script, or a
-Homebrew-managed installation of PostgreSQL will take care of it. See the
-instructions above for more details.
-
-## Documentation
-
-- [Infrastructure overview](./docs/all.md#infrastructure-overview)
-  - [GitHub Repositories](./docs/all.md#github-repositories)
-  - [Terraform](./docs/all.md#terraform)
-  - [AWS](./docs/all.md#aws)
-  - [New Relic](./docs/all.md#new-relic)
-  - [Onboarding](./docs/all.md#onboarding)
-  - [Setting up the infrastructure](./docs/all.md#setting-up-the-infrastructure)
-- [Using the logs](./docs/all.md#using-the-logs)
-- [Testing](./docs/all.md#testing)
-  - [CI testing](./docs/all.md#ci-testing)
-  - [Manual testing](./docs/all.md#manual-testing)
-  - [To run a local OWASP scan](./docs/all.md#to-run-a-local-owasp-scan)
-  - [End-to-end testing](./docs/all.md#end-to-end-testing)
-- [Deploying](./docs/all.md#deploying)
-  - [Egress Proxy](./docs/all.md#egress-proxy)
-  - [Managing environment variables](./docs/all.md#managing-environment-variables)
-  - [Managing application initialization](./docs/all.md#managing-application-initialization)
-  - [Sandbox environment](./docs/all.md#sandbox-environment)
-- [Database management](./docs/all.md#database-management)
-  - [Initial state](./docs/all.md#initial-state)
-  - [Data Model Diagram](./docs/all.md#data-model-diagram)
-  - [Migrations](./docs/all.md#migrations)
-  - [Purging user data](./docs/all.md#purging-user-data)
-- [One-off tasks](./docs/all.md#one-off-tasks)
-- [Test Loading Commands](./docs/all.md#commands-for-test-loading-the-local-dev-database)
-- [How messages are queued and sent](./docs/all.md#how-messages-are-queued-and-sent)
-- [Writing public APIs](./docs/all.md#writing-public-apis)
-  - [Overview](./docs/all.md#overview)
-  - [Documenting APIs](./docs/all.md#documenting-apis)
-  - [New APIs](./docs/all.md#new-apis)
-- [API Usage](./docs/all.md#api-usage)
-  - [Connecting to the API](./docs/all.md#connecting-to-the-api)
-  - [Postman Documentation](./docs/all.md#postman-documentation)
-  - [Using OpenAPI documentation](./docs/all.md#using-openapi-documentation)
-- [Queues and tasks](./docs/all.md#queues-and-tasks)
-  - [Priority queue](./docs/all.md#priority-queue)
-  - [Celery scheduled tasks](./docs/all.md#celery-scheduled-tasks)
-- [Notify.gov](./docs/all.md#notifygov)
-  - [System Description](./docs/all.md#system-description)
-- [Pull Requests](.docs/all.md#pull-requests)
-  - [Getting Started](.docs/all.md#getting-started)
-  - [Description](.docs/all.md#description)
-  - [TODO (optional)](.docs/all.md#todo-(optional))
-  - [Security Considerations](.docs/all.md#security-considerations)
-- [Code Reviews](.docs/all.md#code-reviews)
-  - [For the reviewer](.docs/all.md#for-the-reviewer)
-  - [For the author](.docs/all.md#for-the-author)
-- [Run Book](./docs/all.md#run-book)
-  - [ Alerts, Notifications, Monitoring](./docs/all.md#-alerts-notifications-monitoring)
-  - [ Restaging Apps](./docs/all.md#-restaging-apps)
-  - [ Smoke-testing the App](./docs/all.md#-smoke-testing-the-app)
-  - [ Configuration Management](./docs/all.md#-configuration-management)
-  - [ DNS Changes](./docs/all.md#-dns-changes)
-  - [Exporting test results for compliance monitoring](./docs/all.md#exporting-test-results-for-compliance-monitoring)
-  - [ Known Gotchas](./docs/all.md#-known-gotchas)
-  - [ User Account Management](./docs/all.md#-user-account-management)
-  - [ SMS Phone Number Management](./docs/all.md#-sms-phone-number-management)
-- [Data Storage Policies \& Procedures](./docs/all.md#data-storage-policies--procedures)
-  - [Potential PII Locations](./docs/all.md#potential-pii-locations)
-  - [Data Retention Policy](./docs/all.md#data-retention-policy)
+A direct installation of PostgreSQL will not put the `createdb` command on your `$PATH`. It can be added there in your shell startup script, or a Homebrew-managed installation of PostgreSQL will take care of it.
 
 ## License && public domain
 
-Work through
-[commit `e604385`](https://github.com/GSA/notifications-api/commit/e604385e0cf4c2ab8c6451b7120ceb196cce21b5)
-is licensed by the UK government under the MIT license. Work after that commit
-is in the worldwide public domain. See [LICENSE.md](./LICENSE.md) for more
-information.
+Work through [commit `e604385`](https://github.com/GSA/notifications-api/commit/e604385e0cf4c2ab8c6451b7120ceb196cce21b5) is licensed by the UK government under the MIT license. Work after that commit is in the worldwide public domain. See [LICENSE.md](./LICENSE.md) for more information.
 
 ## Contributing
 
-As stated in [CONTRIBUTING.md](CONTRIBUTING.md), all contributions to this
-project will be released under the CC0 dedication. By submitting a pull request,
-you are agreeing to comply with this waiver of copyright interest.
+As stated in [CONTRIBUTING.md](CONTRIBUTING.md), all contributions to this project will be released under the CC0 dedication. By submitting a pull request, you are agreeing to comply with this waiver of copyright interest.

app/__init__.py

@@ -3,15 +3,27 @@ import secrets
 import string
 import time
 import uuid
-from contextlib import contextmanager
 from time import monotonic
 
-from celery import Celery, Task, current_task
-from flask import current_app, g, has_request_context, jsonify, make_response, request
-from flask.ctx import has_app_context
+from celery import current_task
+from flask import (
+    current_app,
+    g,
+    has_request_context,
+    jsonify,
+    make_response,
+    request,
+)
 from flask_marshmallow import Marshmallow
 from flask_migrate import Migrate
 from flask_sqlalchemy import SQLAlchemy as _SQLAlchemy
+from gds_metrics import GDSMetrics
+from gds_metrics.metrics import Gauge, Histogram
+from notifications_utils import logging, request_helper
+from notifications_utils.celery import NotifyCelery
+from notifications_utils.clients.encryption.encryption_client import Encryption
+from notifications_utils.clients.redis.redis_client import RedisClient
+from notifications_utils.clients.zendesk.zendesk_client import ZendeskClient
 from sqlalchemy import event
 from werkzeug.exceptions import HTTPException as WerkzeugHTTPException
 from werkzeug.local import LocalProxy
@@ -22,29 +34,6 @@ from app.clients.document_download import DocumentDownloadClient
 from app.clients.email.aws_ses import AwsSesClient
 from app.clients.email.aws_ses_stub import AwsSesStubClient
 from app.clients.sms.aws_sns import AwsSnsClient
-from notifications_utils import logging, request_helper
-from notifications_utils.clients.encryption.encryption_client import Encryption
-from notifications_utils.clients.redis.redis_client import RedisClient
-from notifications_utils.clients.zendesk.zendesk_client import ZendeskClient
-
-
-class NotifyCelery(Celery):
-    def init_app(self, app):
-        self.task_cls = make_task(app)
-
-        # Configure Celery app with options from the main app config.
-        self.config_from_object(app.config["CELERY"])
-
-    def send_task(self, name, args=None, kwargs=None, **other_kwargs):
-        other_kwargs["headers"] = other_kwargs.get("headers") or {}
-
-        if has_request_context() and hasattr(request, "request_id"):
-            other_kwargs["headers"]["notify_request_id"] = request.request_id
-
-        elif has_app_context() and "request_id" in g:
-            other_kwargs["headers"]["notify_request_id"] = g.request_id
-
-        return super().send_task(name, args, kwargs, **other_kwargs)
 
 
 class SQLAlchemy(_SQLAlchemy):
@@ -52,10 +41,10 @@ class SQLAlchemy(_SQLAlchemy):
 
     def apply_driver_hacks(self, app, info, options):
         sa_url, options = super().apply_driver_hacks(app, info, options)
-        if "connect_args" not in options:
-            options["connect_args"] = {}
-        options["connect_args"]["options"] = "-c statement_timeout={}".format(
-            int(app.config["SQLALCHEMY_STATEMENT_TIMEOUT"]) * 1000
+        if 'connect_args' not in options:
+            options['connect_args'] = {}
+        options['connect_args']["options"] = "-c statement_timeout={}".format(
+            int(app.config['SQLALCHEMY_STATEMENT_TIMEOUT']) * 1000
         )
         return (sa_url, options)
 
@@ -72,24 +61,31 @@ encryption = Encryption()
 zendesk_client = ZendeskClient()
 redis_store = RedisClient()
 document_download_client = DocumentDownloadClient()
-
+metrics = GDSMetrics()
 
 notification_provider_clients = NotificationProviderClients()
 
 api_user = LocalProxy(lambda: g.api_user)
 authenticated_service = LocalProxy(lambda: g.authenticated_service)
 
+CONCURRENT_REQUESTS = Gauge(
+    'concurrent_web_request_count',
+    'How many concurrent requests are currently being served',
+)
+
 
 def create_app(application):
     from app.config import configs
 
-    notify_environment = os.environ["NOTIFY_ENVIRONMENT"]
+    notify_environment = os.environ['NOTIFY_ENVIRONMENT']
 
     application.config.from_object(configs[notify_environment])
 
-    application.config["NOTIFY_APP_NAME"] = application.name
+    application.config['NOTIFY_APP_NAME'] = application.name
     init_app(application)
 
+    # Metrics intentionally high up to give the most accurate timing and reliability that the metric is recorded
+    metrics.init_app(application)
     request_helper.init_app(application)
     db.init_app(application)
     migrate.init_app(application, db=db)
@@ -99,16 +95,15 @@ def create_app(application):
     aws_sns_client.init_app(application)
 
     aws_ses_client.init_app()
-    aws_ses_stub_client.init_app(stub_url=application.config["SES_STUB_URL"])
+    aws_ses_stub_client.init_app(
+        stub_url=application.config['SES_STUB_URL']
+    )
     aws_cloudwatch_client.init_app(application)
     # If a stub url is provided for SES, then use the stub client rather than the real SES boto client
-    email_clients = (
-        [aws_ses_stub_client]
-        if application.config["SES_STUB_URL"]
-        else [aws_ses_client]
-    )
+    email_clients = [aws_ses_stub_client] if application.config['SES_STUB_URL'] else [aws_ses_client]
     notification_provider_clients.init_app(
-        sms_clients=[aws_sns_client], email_clients=email_clients
+        sms_clients=[aws_sns_client],
+        email_clients=email_clients
     )
 
     notify_celery.init_app(application)
@@ -117,10 +112,10 @@ def create_app(application):
     document_download_client.init_app(application)
 
     register_blueprint(application)
+    register_v2_blueprints(application)
 
     # avoid circular imports by importing this file later
     from app.commands import setup_commands
-
     setup_commands(application)
 
     # set up sqlalchemy events
@@ -143,17 +138,25 @@ def register_blueprint(application):
     from app.inbound_number.rest import inbound_number_blueprint
     from app.inbound_sms.rest import inbound_sms as inbound_sms_blueprint
     from app.job.rest import job_blueprint
-    from app.notifications.notifications_ses_callback import ses_callback_blueprint
-    from app.notifications.receive_notifications import receive_notifications_blueprint
+    from app.notifications.notifications_ses_callback import (
+        ses_callback_blueprint,
+    )
+    from app.notifications.receive_notifications import (
+        receive_notifications_blueprint,
+    )
     from app.notifications.rest import notifications as notifications_blueprint
     from app.organization.invite_rest import organization_invite_blueprint
     from app.organization.rest import organization_blueprint
     from app.performance_dashboard.rest import performance_dashboard_blueprint
     from app.platform_stats.rest import platform_stats_blueprint
-    from app.provider_details.rest import provider_details as provider_details_blueprint
+    from app.provider_details.rest import (
+        provider_details as provider_details_blueprint,
+    )
     from app.service.callback_rest import service_callback_blueprint
     from app.service.rest import service_blueprint
-    from app.service_invite.rest import service_invite as service_invite_blueprint
+    from app.service_invite.rest import (
+        service_invite as service_invite_blueprint,
+    )
     from app.status.healthcheck import status as status_blueprint
     from app.template.rest import template_blueprint
     from app.template_folder.rest import template_folder_blueprint
@@ -165,10 +168,10 @@ def register_blueprint(application):
     from app.webauthn.rest import webauthn_blueprint
 
     service_blueprint.before_request(requires_admin_auth)
-    application.register_blueprint(service_blueprint, url_prefix="/service")
+    application.register_blueprint(service_blueprint, url_prefix='/service')
 
     user_blueprint.before_request(requires_admin_auth)
-    application.register_blueprint(user_blueprint, url_prefix="/user")
+    application.register_blueprint(user_blueprint, url_prefix='/user')
 
     webauthn_blueprint.before_request(requires_admin_auth)
     application.register_blueprint(webauthn_blueprint)
@@ -215,14 +218,10 @@ def register_blueprint(application):
     application.register_blueprint(events_blueprint)
 
     provider_details_blueprint.before_request(requires_admin_auth)
-    application.register_blueprint(
-        provider_details_blueprint, url_prefix="/provider-details"
-    )
+    application.register_blueprint(provider_details_blueprint, url_prefix='/provider-details')
 
     email_branding_blueprint.before_request(requires_admin_auth)
-    application.register_blueprint(
-        email_branding_blueprint, url_prefix="/email-branding"
-    )
+    application.register_blueprint(email_branding_blueprint, url_prefix='/email-branding')
 
     billing_blueprint.before_request(requires_admin_auth)
     application.register_blueprint(billing_blueprint)
@@ -231,7 +230,7 @@ def register_blueprint(application):
     application.register_blueprint(service_callback_blueprint)
 
     organization_blueprint.before_request(requires_admin_auth)
-    application.register_blueprint(organization_blueprint, url_prefix="/organizations")
+    application.register_blueprint(organization_blueprint, url_prefix='/organizations')
 
     complaint_blueprint.before_request(requires_admin_auth)
     application.register_blueprint(complaint_blueprint)
@@ -240,9 +239,7 @@ def register_blueprint(application):
     application.register_blueprint(performance_dashboard_blueprint)
 
     platform_stats_blueprint.before_request(requires_admin_auth)
-    application.register_blueprint(
-        platform_stats_blueprint, url_prefix="/platform-stats"
-    )
+    application.register_blueprint(platform_stats_blueprint, url_prefix='/platform-stats')
 
     template_folder_blueprint.before_request(requires_admin_auth)
     application.register_blueprint(template_folder_blueprint)
@@ -251,26 +248,60 @@ def register_blueprint(application):
     application.register_blueprint(upload_blueprint)
 
 
+def register_v2_blueprints(application):
+    from app.authentication.auth import requires_auth
+    from app.v2.inbound_sms.get_inbound_sms import v2_inbound_sms_blueprint
+    from app.v2.notifications import (  # noqa
+        get_notifications,
+        post_notifications,
+        v2_notification_blueprint,
+    )
+    from app.v2.template import (  # noqa
+        get_template,
+        post_template,
+        v2_template_blueprint,
+    )
+    from app.v2.templates.get_templates import v2_templates_blueprint
+
+    v2_notification_blueprint.before_request(requires_auth)
+    application.register_blueprint(v2_notification_blueprint)
+
+    v2_templates_blueprint.before_request(requires_auth)
+    application.register_blueprint(v2_templates_blueprint)
+
+    v2_template_blueprint.before_request(requires_auth)
+    application.register_blueprint(v2_template_blueprint)
+
+    v2_inbound_sms_blueprint.before_request(requires_auth)
+    application.register_blueprint(v2_inbound_sms_blueprint)
+
+
 def init_app(app):
 
     @app.before_request
     def record_request_details():
+        CONCURRENT_REQUESTS.inc()
+
         g.start = monotonic()
         g.endpoint = request.endpoint
 
     @app.after_request
     def after_request(response):
-        response.headers.add("X-Content-Type-Options", "nosniff")
+        CONCURRENT_REQUESTS.dec()
+
+        response.headers.add('X-Content-Type-Options', 'nosniff')
         return response
 
     @app.errorhandler(Exception)
     def exception(error):
         app.logger.exception(error)
         # error.code is set for our exception types.
-        msg = getattr(error, "message", str(error))
-        code = getattr(error, "code", 500)
+        msg = getattr(error, 'message', str(error))
+        code = getattr(error, 'code', 500)
         response = make_response(
-            jsonify(result="error", message=msg), code, error.get_headers()
+            jsonify(result='error', message=msg),
+            code,
+            error.get_headers()
         )
         response.content_type = "application/json"
         return response
@@ -278,18 +309,22 @@ def init_app(app):
     @app.errorhandler(WerkzeugHTTPException)
     def werkzeug_exception(e):
         response = make_response(
-            jsonify(result="error", message=e.description), e.code, e.get_headers()
+            jsonify(result='error', message=e.description),
+            e.code,
+            e.get_headers()
         )
-        response.content_type = "application/json"
+        response.content_type = 'application/json'
         return response
 
     @app.errorhandler(404)
     def page_not_found(e):
         msg = e.description or "Not found"
         response = make_response(
-            jsonify(result="error", message=msg), 404, e.get_headers()
+            jsonify(result='error', message=msg),
+            404,
+            e.get_headers()
         )
-        response.content_type = "application/json"
+        response.content_type = 'application/json'
         return response
 
 
@@ -298,29 +333,47 @@ def create_uuid():
 
 
 def create_random_identifier():
-    return "".join(
-        secrets.choice(string.ascii_uppercase + string.digits) for _ in range(16)
+    return ''.join(secrets.choice(string.ascii_uppercase + string.digits) for _ in range(16))
+
+
+def setup_sqlalchemy_events(app):
+
+    TOTAL_DB_CONNECTIONS = Gauge(
+        'db_connection_total_connected',
+        'How many db connections are currently held (potentially idle) by the server',
     )
 
+    TOTAL_CHECKED_OUT_DB_CONNECTIONS = Gauge(
+        'db_connection_total_checked_out',
+        'How many db connections are currently checked out by web requests',
+    )
+
+    DB_CONNECTION_OPEN_DURATION_SECONDS = Histogram(
+        'db_connection_open_duration_seconds',
+        'How long db connections are held open for in seconds',
+        ['method', 'host', 'path']
+    )
 
-# TODO maintainability what is the purpose of this?  Debugging?
-def setup_sqlalchemy_events(app):
     # need this or db.engine isn't accessible
     with app.app_context():
+        @event.listens_for(db.engine, 'connect')
+        def connect(dbapi_connection, connection_record):
+            # connection first opened with db
+            TOTAL_DB_CONNECTIONS.inc()
 
-        @event.listens_for(db.engine, "connect")
-        def connect(dbapi_connection, connection_record):  # noqa
-            pass
+        @event.listens_for(db.engine, 'close')
+        def close(dbapi_connection, connection_record):
+            # connection closed (probably only happens with overflow connections)
+            TOTAL_DB_CONNECTIONS.dec()
 
-        @event.listens_for(db.engine, "close")
-        def close(dbapi_connection, connection_record):  # noqa
-            pass
-
-        @event.listens_for(db.engine, "checkout")
-        def checkout(dbapi_connection, connection_record, connection_proxy):  # noqa
+        @event.listens_for(db.engine, 'checkout')
+        def checkout(dbapi_connection, connection_record, connection_proxy):
             try:
+                # connection given to a web worker
+                TOTAL_CHECKED_OUT_DB_CONNECTIONS.inc()
+
                 # this will overwrite any previous checkout_at timestamp
-                connection_record.info["checkout_at"] = time.monotonic()
+                connection_record.info['checkout_at'] = time.monotonic()
 
                 # checkin runs after the request is already torn down, therefore we add the request_data onto the
                 # connection_record as otherwise it won't have that information when checkin actually runs.
@@ -328,88 +381,42 @@ def setup_sqlalchemy_events(app):
 
                 # web requests
                 if has_request_context():
-                    connection_record.info["request_data"] = {
-                        "method": request.method,
-                        "host": request.host,
-                        "url_rule": (
-                            request.url_rule.rule if request.url_rule else "No endpoint"
-                        ),
+                    connection_record.info['request_data'] = {
+                        'method': request.method,
+                        'host': request.host,
+                        'url_rule': request.url_rule.rule if request.url_rule else 'No endpoint'
                     }
                 # celery apps
                 elif current_task:
-                    connection_record.info["request_data"] = {
-                        "method": "celery",
-                        "host": current_app.config["NOTIFY_APP_NAME"],  # worker name
-                        "url_rule": current_task.name,  # task name
+                    connection_record.info['request_data'] = {
+                        'method': 'celery',
+                        'host': current_app.config['NOTIFY_APP_NAME'],  # worker name
+                        'url_rule': current_task.name,  # task name
                     }
                 # anything else. migrations possibly, or flask cli commands.
                 else:
-                    current_app.logger.warning(
-                        "Checked out sqlalchemy connection from outside of request/task"
-                    )
-                    connection_record.info["request_data"] = {
-                        "method": "unknown",
-                        "host": "unknown",
-                        "url_rule": "unknown",
+                    current_app.logger.warning('Checked out sqlalchemy connection from outside of request/task')
+                    connection_record.info['request_data'] = {
+                        'method': 'unknown',
+                        'host': 'unknown',
+                        'url_rule': 'unknown',
                     }
             except Exception:
                 current_app.logger.exception("Exception caught for checkout event.")
 
-        @event.listens_for(db.engine, "checkin")
-        def checkin(dbapi_connection, connection_record):  # noqa
-            pass
+        @event.listens_for(db.engine, 'checkin')
+        def checkin(dbapi_connection, connection_record):
+            try:
+                # connection returned by a web worker
+                TOTAL_CHECKED_OUT_DB_CONNECTIONS.dec()
 
+                # duration that connection was held by a single web request
+                duration = time.monotonic() - connection_record.info['checkout_at']
 
-def make_task(app):
-    class NotifyTask(Task):
-        abstract = True
-        start = None
-
-        @property
-        def queue_name(self):
-            delivery_info = self.request.delivery_info or {}
-            return delivery_info.get("routing_key", "none")
-
-        @property
-        def request_id(self):
-            # Note that each header is a direct attribute of the
-            # task context (aka "request").
-            return self.request.get("notify_request_id")
-
-        @contextmanager
-        def app_context(self):
-            with app.app_context():
-                # Add 'request_id' to 'g' so that it gets logged.
-                g.request_id = self.request_id
-                yield
-
-        def on_success(self, retval, task_id, args, kwargs):  # noqa
-            # enables request id tracing for these logs
-            with self.app_context():
-                elapsed_time = time.monotonic() - self.start
-
-                app.logger.info(
-                    "Celery task {task_name} (queue: {queue_name}) took {time}".format(
-                        task_name=self.name,
-                        queue_name=self.queue_name,
-                        time="{0:.4f}".format(elapsed_time),
-                    )
-                )
-
-        def on_failure(self, exc, task_id, args, kwargs, einfo):  # noqa
-            # enables request id tracing for these logs
-            with self.app_context():
-                app.logger.exception(
-                    "Celery task {task_name} (queue: {queue_name}) failed".format(
-                        task_name=self.name,
-                        queue_name=self.queue_name,
-                    )
-                )
-
-        def __call__(self, *args, **kwargs):
-            # ensure task has flask context to access config, logger, etc
-            with self.app_context():
-                self.start = time.monotonic()
-                return super().__call__(*args, **kwargs)
-
-    return NotifyTask
+                DB_CONNECTION_OPEN_DURATION_SECONDS.labels(
+                    connection_record.info['request_data']['method'],
+                    connection_record.info['request_data']['host'],
+                    connection_record.info['request_data']['url_rule']
+                ).observe(duration)
+            except Exception:
+                current_app.logger.exception("Exception caught for checkin event.")

app/authentication/auth.py

@@ -1,6 +1,7 @@
 import uuid
 
 from flask import current_app, g, request
+from gds_metrics import Histogram
 from notifications_python_client.authentication import (
     decode_jwt_token,
     get_token_issuer,
@@ -12,19 +13,22 @@ from notifications_python_client.errors import (
     TokenExpiredError,
     TokenIssuerError,
 )
+from notifications_utils import request_helper
 from sqlalchemy.orm.exc import NoResultFound
 
 from app.serialised_models import SerialisedService
-from notifications_utils import request_helper
 
 # stvnrlly - this is silly, but bandit has a multiline string bug (https://github.com/PyCQA/bandit/issues/658)
 # and flake8 wants a multiline quote here. TODO: check on bug status and restore sanity once possible
-TOKEN_MESSAGE_ONE = (
-    "Invalid token: make sure your API token matches the example "  # nosec B105
-)
+TOKEN_MESSAGE_ONE = "Invalid token: make sure your API token matches the example "  # nosec B105
 TOKEN_MESSAGE_TWO = "at https://docs.notifications.service.gov.uk/rest-api.html#authorisation-header"  # nosec B105
 GENERAL_TOKEN_ERROR_MESSAGE = TOKEN_MESSAGE_ONE + TOKEN_MESSAGE_TWO
 
+AUTH_DB_CONNECTION_DURATION_SECONDS = Histogram(
+    'auth_db_connection_duration_seconds',
+    'Time taken to get DB connection and fetch service from database',
+)
+
 
 class AuthError(Exception):
     def __init__(self, message, code, service_id=None, api_key_id=None):
@@ -35,18 +39,21 @@ class AuthError(Exception):
         self.api_key_id = api_key_id
 
     def __str__(self):
-        return "AuthError({message}, {code}, service_id={service_id}, api_key_id={api_key_id})".format(
-            **self.__dict__
-        )
+        return 'AuthError({message}, {code}, service_id={service_id}, api_key_id={api_key_id})'.format(**self.__dict__)
 
     def to_dict_v2(self):
         return {
-            "status_code": self.code,
-            "errors": [{"error": "AuthError", "message": self.short_message}],
+            'status_code': self.code,
+            "errors": [
+                {
+                    "error": "AuthError",
+                    "message": self.short_message
+                }
+            ]
         }
 
 
-class InternalApiKey:
+class InternalApiKey():
     def __init__(self, client_id, secret):
         self.secret = secret
         self.id = client_id
@@ -58,11 +65,11 @@ def requires_no_auth():
 
 
 def requires_admin_auth():
-    requires_internal_auth(current_app.config.get("ADMIN_CLIENT_ID"))
+    requires_internal_auth(current_app.config.get('ADMIN_CLIENT_ID'))
 
 
 def requires_internal_auth(expected_client_id):
-    if expected_client_id not in current_app.config.get("INTERNAL_CLIENT_API_KEYS"):
+    if expected_client_id not in current_app.config.get('INTERNAL_CLIENT_API_KEYS'):
         raise TypeError("Unknown client_id for internal auth")
 
     request_helper.check_proxy_header_before_request()
@@ -70,13 +77,13 @@ def requires_internal_auth(expected_client_id):
     client_id = _get_token_issuer(auth_token)
 
     if client_id != expected_client_id:
-        current_app.logger.info("client_id: %s", client_id)
-        current_app.logger.info("expected_client_id: %s", expected_client_id)
+        current_app.logger.info('client_id: %s', client_id)
+        current_app.logger.info('expected_client_id: %s', expected_client_id)
         raise AuthError("Unauthorized: not allowed to perform this action", 401)
 
     api_keys = [
         InternalApiKey(client_id, secret)
-        for secret in current_app.config.get("INTERNAL_CLIENT_API_KEYS")[client_id]
+        for secret in current_app.config.get('INTERNAL_CLIENT_API_KEYS')[client_id]
     ]
 
     _decode_jwt_token(auth_token, api_keys, client_id)
@@ -87,9 +94,7 @@ def requires_auth():
     request_helper.check_proxy_header_before_request()
 
     auth_token = _get_auth_token(request)
-    issuer = _get_token_issuer(
-        auth_token
-    )  # ie the `iss` claim which should be a service ID
+    issuer = _get_token_issuer(auth_token)  # ie the `iss` claim which should be a service ID
 
     try:
         service_id = uuid.UUID(issuer)
@@ -97,27 +102,25 @@ def requires_auth():
         raise AuthError("Invalid token: service id is not the right data type", 403)
 
     try:
-        service = SerialisedService.from_id(service_id)
+        with AUTH_DB_CONNECTION_DURATION_SECONDS.time():
+            service = SerialisedService.from_id(service_id)
     except NoResultFound:
         raise AuthError("Invalid token: service not found", 403)
 
     if not service.api_keys:
-        raise AuthError(
-            "Invalid token: service has no API keys", 403, service_id=service.id
-        )
+        raise AuthError("Invalid token: service has no API keys", 403, service_id=service.id)
 
     if not service.active:
-        raise AuthError(
-            "Invalid token: service is archived", 403, service_id=service.id
-        )
+        raise AuthError("Invalid token: service is archived", 403, service_id=service.id)
 
     api_key = _decode_jwt_token(auth_token, service.api_keys, service.id)
 
-    current_app.logger.info(
-        "API authorised for service {} with api key {}, using issuer {} for URL: {}".format(
-            service_id, api_key.id, request.headers.get("User-Agent"), request.base_url
-        )
-    )
+    current_app.logger.info('API authorised for service {} with api key {}, using issuer {} for URL: {}'.format(
+        service_id,
+        api_key.id,
+        request.headers.get('User-Agent'),
+        request.base_url
+    ))
 
     g.api_user = api_key
     g.service_id = service_id
@@ -129,13 +132,9 @@ def _decode_jwt_token(auth_token, api_keys, service_id=None):
         try:
             decode_jwt_token(auth_token, api_key.secret)
         except TokenExpiredError:
-            if not current_app.config.get("ALLOW_EXPIRED_API_TOKEN", False):
-                err_msg = (
-                    "Error: Your system clock must be accurate to within 30 seconds"
-                )
-                raise AuthError(
-                    err_msg, 403, service_id=service_id, api_key_id=api_key.id
-                )
+            if not current_app.config.get('ALLOW_EXPIRED_API_TOKEN', False):
+                err_msg = "Error: Your system clock must be accurate to within 30 seconds"
+                raise AuthError(err_msg, 403, service_id=service_id, api_key_id=api_key.id)
         except TokenAlgorithmError:
             err_msg = "Invalid token: algorithm used is not HS256"
             raise AuthError(err_msg, 403, service_id=service_id, api_key_id=api_key.id)
@@ -148,20 +147,10 @@ def _decode_jwt_token(auth_token, api_keys, service_id=None):
             continue
         except TokenError:
             # General error when trying to decode and validate the token
-            raise AuthError(
-                GENERAL_TOKEN_ERROR_MESSAGE,
-                403,
-                service_id=service_id,
-                api_key_id=api_key.id,
-            )
+            raise AuthError(GENERAL_TOKEN_ERROR_MESSAGE, 403, service_id=service_id, api_key_id=api_key.id)
 
         if api_key.expiry_date:
-            raise AuthError(
-                "Invalid token: API key revoked",
-                403,
-                service_id=service_id,
-                api_key_id=api_key.id,
-            )
+            raise AuthError("Invalid token: API key revoked", 403, service_id=service_id, api_key_id=api_key.id)
 
         return api_key
     else:
@@ -170,14 +159,14 @@ def _decode_jwt_token(auth_token, api_keys, service_id=None):
 
 
 def _get_auth_token(req):
-    auth_header = req.headers.get("Authorization", None)
+    auth_header = req.headers.get('Authorization', None)
     if not auth_header:
-        raise AuthError("Unauthorized: authentication token must be provided", 401)
+        raise AuthError('Unauthorized: authentication token must be provided', 401)
 
     auth_scheme = auth_header[:7].title()
 
-    if auth_scheme != "Bearer ":
-        raise AuthError("Unauthorized: authentication bearer scheme must be used", 401)
+    if auth_scheme != 'Bearer ':
+        raise AuthError('Unauthorized: authentication bearer scheme must be used', 401)
 
     return auth_header[7:]
 

app/aws/s3.py

@@ -1,309 +1,61 @@
-import re
-
 import botocore
 from boto3 import Session
-from expiringdict import ExpiringDict
 from flask import current_app
 
-from app import redis_store
-from app.clients import AWS_CLIENT_CONFIG
-
-FILE_LOCATION_STRUCTURE = "service-{}-notify/{}.csv"
-
-# Temporarily extend cache to 7 days
-ttl = 60 * 60 * 24 * 7
-JOBS = ExpiringDict(max_len=20000, max_age_seconds=ttl)
+FILE_LOCATION_STRUCTURE = 'service-{}-notify/{}.csv'
 
 
-JOBS_CACHE_HITS = "JOBS_CACHE_HITS"
-JOBS_CACHE_MISSES = "JOBS_CACHE_MISSES"
-
-
-def list_s3_objects():
-    bucket_name = current_app.config["CSV_UPLOAD_BUCKET"]["bucket"]
-    access_key = current_app.config["CSV_UPLOAD_BUCKET"]["access_key_id"]
-    secret_key = current_app.config["CSV_UPLOAD_BUCKET"]["secret_access_key"]
-    region = current_app.config["CSV_UPLOAD_BUCKET"]["region"]
-    session = Session(
-        aws_access_key_id=access_key,
-        aws_secret_access_key=secret_key,
-        region_name=region,
-    )
-    s3 = session.client("s3")
-
-    try:
-        response = s3.list_objects_v2(Bucket=bucket_name)
-        while True:
-            for obj in response.get("Contents", []):
-                yield obj["Key"]
-            if "NextContinuationToken" in response:
-                response = s3.list_objects_v2(
-                    Bucket=bucket_name,
-                    ContinuationToken=response["NextContinuationToken"],
-                )
-            else:
-                break
-    except Exception as e:
-        current_app.logger.error(
-            f"An error occurred while regenerating cache #notify-admin-1200 {e}"
-        )
-
-
-def get_s3_files():
-    current_app.logger.info("Regenerate job cache #notify-admin-1200")
-    bucket_name = current_app.config["CSV_UPLOAD_BUCKET"]["bucket"]
-    access_key = current_app.config["CSV_UPLOAD_BUCKET"]["access_key_id"]
-    secret_key = current_app.config["CSV_UPLOAD_BUCKET"]["secret_access_key"]
-    region = current_app.config["CSV_UPLOAD_BUCKET"]["region"]
-    session = Session(
-        aws_access_key_id=access_key,
-        aws_secret_access_key=secret_key,
-        region_name=region,
-    )
-    objects = list_s3_objects()
-
-    s3res = session.resource("s3", config=AWS_CLIENT_CONFIG)
-    current_app.logger.info(
-        f"JOBS cache length before regen: {len(JOBS)} #notify-admin-1200"
-    )
-    for object in objects:
-        # We put our csv files in the format "service-{service_id}-notify/{job_id}"
-        try:
-            object_arr = object.split("/")
-            job_id = object_arr[1]  # get the job_id
-            job_id = job_id.replace(".csv", "")  # we just want the job_id
-            if JOBS.get(job_id) is None:
-                object = (
-                    s3res.Object(bucket_name, object)
-                    .get()["Body"]
-                    .read()
-                    .decode("utf-8")
-                )
-                if "phone number" in object.lower():
-                    JOBS[job_id] = object
-        except LookupError as le:
-            # perhaps our key is not formatted as we expected.  If so skip it.
-            current_app.logger.error(f"LookupError {le} #notify-admin-1200")
-
-    current_app.logger.info(
-        f"JOBS cache length after regen: {len(JOBS)} #notify-admin-1200"
-    )
-
-
-def get_s3_file(bucket_name, file_location, access_key, secret_key, region):
+def get_s3_file(
+    bucket_name, file_location, access_key, secret_key, region
+):
     s3_file = get_s3_object(bucket_name, file_location, access_key, secret_key, region)
-    return s3_file.get()["Body"].read().decode("utf-8")
+    return s3_file.get()['Body'].read().decode('utf-8')
 
 
-def get_s3_object(bucket_name, file_location, access_key, secret_key, region):
-    session = Session(
-        aws_access_key_id=access_key,
-        aws_secret_access_key=secret_key,
-        region_name=region,
-    )
-    s3 = session.resource("s3", config=AWS_CLIENT_CONFIG)
+def get_s3_object(
+    bucket_name, file_location, access_key, secret_key, region
+):
+    session = Session(aws_access_key_id=access_key, aws_secret_access_key=secret_key, region_name=region)
+    s3 = session.resource('s3')
     return s3.Object(bucket_name, file_location)
 
 
-def purge_bucket(bucket_name, access_key, secret_key, region):
-    session = Session(
-        aws_access_key_id=access_key,
-        aws_secret_access_key=secret_key,
-        region_name=region,
-    )
-    s3 = session.resource("s3", config=AWS_CLIENT_CONFIG)
-    bucket = s3.Bucket(bucket_name)
-    bucket.objects.all().delete()
-
-
-def file_exists(bucket_name, file_location, access_key, secret_key, region):
+def file_exists(
+    bucket_name, file_location, access_key, secret_key, region
+):
     try:
         # try and access metadata of object
-        get_s3_object(
-            bucket_name, file_location, access_key, secret_key, region
-        ).metadata
+        get_s3_object(bucket_name, file_location, access_key, secret_key, region).metadata
         return True
     except botocore.exceptions.ClientError as e:
-        if e.response["ResponseMetadata"]["HTTPStatusCode"] == 404:
+        if e.response['ResponseMetadata']['HTTPStatusCode'] == 404:
             return False
         raise
 
 
 def get_job_location(service_id, job_id):
     return (
-        current_app.config["CSV_UPLOAD_BUCKET"]["bucket"],
+        current_app.config['CSV_UPLOAD_BUCKET']['bucket'],
         FILE_LOCATION_STRUCTURE.format(service_id, job_id),
-        current_app.config["CSV_UPLOAD_BUCKET"]["access_key_id"],
-        current_app.config["CSV_UPLOAD_BUCKET"]["secret_access_key"],
-        current_app.config["CSV_UPLOAD_BUCKET"]["region"],
+        current_app.config['CSV_UPLOAD_BUCKET']['access_key_id'],
+        current_app.config['CSV_UPLOAD_BUCKET']['secret_access_key'],
+        current_app.config['CSV_UPLOAD_BUCKET']['region'],
     )
 
 
 def get_job_and_metadata_from_s3(service_id, job_id):
     obj = get_s3_object(*get_job_location(service_id, job_id))
-    return obj.get()["Body"].read().decode("utf-8"), obj.get()["Metadata"]
+    return obj.get()['Body'].read().decode('utf-8'), obj.get()['Metadata']
 
 
 def get_job_from_s3(service_id, job_id):
     obj = get_s3_object(*get_job_location(service_id, job_id))
-    return obj.get()["Body"].read().decode("utf-8")
-
-
-def incr_jobs_cache_misses():
-    if not redis_store.get(JOBS_CACHE_MISSES):
-        redis_store.set(JOBS_CACHE_MISSES, 1)
-    else:
-        redis_store.incr(JOBS_CACHE_MISSES)
-
-
-def incr_jobs_cache_hits():
-    if not redis_store.get(JOBS_CACHE_HITS):
-        redis_store.set(JOBS_CACHE_HITS, 1)
-    else:
-        redis_store.incr(JOBS_CACHE_HITS)
-
-
-def extract_phones(job):
-    job = job.split("\r\n")
-    first_row = job[0]
-    job.pop(0)
-    first_row = first_row.split(",")
-    phone_index = 0
-    for item in first_row:
-        # Note: may contain a BOM and look like \ufeffphone number
-        if "phone number" in item.lower():
-            break
-        phone_index = phone_index + 1
-
-    phones = {}
-    job_row = 0
-    for row in job:
-        row = row.split(",")
-
-        if phone_index >= len(row):
-            phones[job_row] = "Unavailable"
-            current_app.logger.error(
-                "Corrupt csv file, missing columns or possibly a byte order mark in the file"
-            )
-
-        else:
-            my_phone = row[phone_index]
-            my_phone = re.sub(r"[\+\s\(\)\-\.]*", "", my_phone)
-            phones[job_row] = my_phone
-        job_row = job_row + 1
-    return phones
-
-
-def extract_personalisation(job):
-    job = job.split("\r\n")
-    first_row = job[0]
-    job.pop(0)
-    first_row = first_row.split(",")
-    personalisation = {}
-    job_row = 0
-    for row in job:
-        row = row.split(",")
-        temp = dict(zip(first_row, row))
-        personalisation[job_row] = temp
-        job_row = job_row + 1
-    return personalisation
-
-
-def get_phone_number_from_s3(service_id, job_id, job_row_number):
-    # We don't want to constantly pull down a job from s3 every time we need a phone number.
-    # At the same time we don't want to store it in redis or the db
-    # So this is a little recycling mechanism to reduce the number of downloads.
-    job = JOBS.get(job_id)
-    if job is None:
-        job = get_job_from_s3(service_id, job_id)
-        JOBS[job_id] = job
-        incr_jobs_cache_misses()
-    else:
-        incr_jobs_cache_hits()
-
-    # If the job is None after our attempt to retrieve it from s3, it
-    # probably means the job is old and has been deleted from s3, in
-    # which case there is nothing we can do.  It's unlikely to run into
-    # this, but it could theoretically happen, especially if we ever
-    # change the task schedules
-    if job is None:
-        current_app.logger.warning(
-            f"Couldnt find phone for job_id {job_id} row number {job_row_number} because job is missing"
-        )
-        return "Unavailable"
-
-    # If we look in the JOBS cache for the quick lookup dictionary of phones for a given job
-    # and that dictionary is not there, create it
-    if JOBS.get(f"{job_id}_phones") is None:
-        JOBS[f"{job_id}_phones"] = extract_phones(job)
-
-    # If we can find the quick dictionary, use it
-    if JOBS.get(f"{job_id}_phones") is not None:
-        phone_to_return = JOBS.get(f"{job_id}_phones").get(job_row_number)
-        if phone_to_return:
-            return phone_to_return
-        else:
-            current_app.logger.warning(
-                f"Was unable to retrieve phone number from lookup dictionary for job {job_id}"
-            )
-            return "Unavailable"
-    else:
-        current_app.logger.error(
-            f"Was unable to construct lookup dictionary for job {job_id}"
-        )
-        return "Unavailable"
-
-
-def get_personalisation_from_s3(service_id, job_id, job_row_number):
-    # We don't want to constantly pull down a job from s3 every time we need the personalisation.
-    # At the same time we don't want to store it in redis or the db
-    # So this is a little recycling mechanism to reduce the number of downloads.
-    job = JOBS.get(job_id)
-    if job is None:
-        job = get_job_from_s3(service_id, job_id)
-        JOBS[job_id] = job
-        incr_jobs_cache_misses()
-    else:
-        incr_jobs_cache_hits()
-
-    # If the job is None after our attempt to retrieve it from s3, it
-    # probably means the job is old and has been deleted from s3, in
-    # which case there is nothing we can do.  It's unlikely to run into
-    # this, but it could theoretically happen, especially if we ever
-    # change the task schedules
-    if job is None:
-        current_app.logger.warning(
-            "Couldnt find personalisation for job_id {job_id} row number {job_row_number} because job is missing"
-        )
-        return {}
-
-    # If we look in the JOBS cache for the quick lookup dictionary of personalisations for a given job
-    # and that dictionary is not there, create it
-    if JOBS.get(f"{job_id}_personalisation") is None:
-        JOBS[f"{job_id}_personalisation"] = extract_personalisation(job)
-
-    # If we can find the quick dictionary, use it
-    if JOBS.get(f"{job_id}_personalisation") is not None:
-        personalisation_to_return = JOBS.get(f"{job_id}_personalisation").get(
-            job_row_number
-        )
-        if personalisation_to_return:
-            return personalisation_to_return
-        else:
-            current_app.logger.warning(
-                f"Was unable to retrieve personalisation from lookup dictionary for job {job_id}"
-            )
-            return {}
-    else:
-        current_app.logger.error(
-            f"Was unable to construct lookup dictionary for job {job_id}"
-        )
-        return {}
+    return obj.get()['Body'].read().decode('utf-8')
 
 
 def get_job_metadata_from_s3(service_id, job_id):
     obj = get_s3_object(*get_job_location(service_id, job_id))
-    return obj.get()["Metadata"]
+    return obj.get()['Metadata']
 
 
 def remove_job_from_s3(service_id, job_id):
@@ -317,10 +69,10 @@ def remove_s3_object(bucket_name, object_key, access_key, secret_key, region):
 
 def remove_csv_object(object_key):
     obj = get_s3_object(
-        current_app.config["CSV_UPLOAD_BUCKET"]["bucket"],
+        current_app.config['CSV_UPLOAD_BUCKET']['bucket'],
         object_key,
-        current_app.config["CSV_UPLOAD_BUCKET"]["access_key_id"],
-        current_app.config["CSV_UPLOAD_BUCKET"]["secret_access_key"],
-        current_app.config["CSV_UPLOAD_BUCKET"]["region"],
+        current_app.config['CSV_UPLOAD_BUCKET']['access_key_id'],
+        current_app.config['CSV_UPLOAD_BUCKET']['secret_access_key'],
+        current_app.config['CSV_UPLOAD_BUCKET']['region']
     )
     return obj.delete()

app/billing/billing_schemas.py

@@ -1,7 +1,5 @@
 from datetime import datetime
 
-from app.enums import NotificationType
-
 create_or_update_free_sms_fragment_limit_schema = {
     "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "POST annual billing schema",
@@ -10,7 +8,7 @@ create_or_update_free_sms_fragment_limit_schema = {
     "properties": {
         "free_sms_fragment_limit": {"type": "integer", "minimum": 0},
     },
-    "required": ["free_sms_fragment_limit"],
+    "required": ["free_sms_fragment_limit"]
 }
 
 
@@ -27,7 +25,7 @@ def serialize_ft_billing_remove_emails(rows):
             "charged_units": row.charged_units,
         }
         for row in rows
-        if row.notification_type != NotificationType.EMAIL
+        if row.notification_type != 'email'
     ]
 
 

app/billing/rest.py

@@ -21,43 +21,44 @@ from app.models import Service
 from app.schema_validation import validate
 
 billing_blueprint = Blueprint(
-    "billing", __name__, url_prefix="/service/<uuid:service_id>/billing"
+    'billing',
+    __name__,
+    url_prefix='/service/<uuid:service_id>/billing'
 )
 
 
 register_errors(billing_blueprint)
 
 
-@billing_blueprint.route("/monthly-usage")
+@billing_blueprint.route('/monthly-usage')
 def get_yearly_usage_by_monthly_from_ft_billing(service_id):
     try:
-        year = int(request.args.get("year"))
+        year = int(request.args.get('year'))
     except TypeError:
-        return jsonify(result="error", message="No valid year provided"), 400
+        return jsonify(result='error', message='No valid year provided'), 400
     results = fetch_monthly_billing_for_year(service_id=service_id, year=year)
     data = serialize_ft_billing_remove_emails(results)
     return jsonify(data)
 
 
-@billing_blueprint.route("/yearly-usage-summary")
+@billing_blueprint.route('/yearly-usage-summary')
 def get_yearly_billing_usage_summary_from_ft_billing(service_id):
     try:
-        year = int(request.args.get("year"))
+        year = int(request.args.get('year'))
     except TypeError:
-        return jsonify(result="error", message="No valid year provided"), 400
+        return jsonify(result='error', message='No valid year provided'), 400
 
     billing_data = fetch_billing_totals_for_year(service_id, year)
     data = serialize_ft_billing_yearly_totals(billing_data)
     return jsonify(data)
 
 
-@billing_blueprint.route("/free-sms-fragment-limit", methods=["GET"])
+@billing_blueprint.route('/free-sms-fragment-limit', methods=["GET"])
 def get_free_sms_fragment_limit(service_id):
-    financial_year_start = request.args.get("financial_year_start")
 
-    annual_billing = dao_get_free_sms_fragment_limit_for_year(
-        service_id, financial_year_start
-    )
+    financial_year_start = request.args.get('financial_year_start')
+
+    annual_billing = dao_get_free_sms_fragment_limit_for_year(service_id, financial_year_start)
 
     if annual_billing is None:
         service = Service.query.get(service_id)
@@ -66,39 +67,40 @@ def get_free_sms_fragment_limit(service_id):
 
         annual_billing = set_default_free_allowance_for_service(
             service=service,
-            year_start=int(financial_year_start) if financial_year_start else None,
+            year_start=int(financial_year_start) if financial_year_start else None
         )
 
     return jsonify(annual_billing.serialize_free_sms_items()), 200
 
 
-@billing_blueprint.route("/free-sms-fragment-limit", methods=["POST"])
+@billing_blueprint.route('/free-sms-fragment-limit', methods=["POST"])
 def create_or_update_free_sms_fragment_limit(service_id):
+
     req_args = request.get_json()
 
     form = validate(req_args, create_or_update_free_sms_fragment_limit_schema)
 
-    update_free_sms_fragment_limit_data(
-        service_id,
-        free_sms_fragment_limit=form.get("free_sms_fragment_limit"),
-        financial_year_start=form.get("financial_year_start"),
-    )
+    update_free_sms_fragment_limit_data(service_id,
+                                        free_sms_fragment_limit=form.get('free_sms_fragment_limit'),
+                                        financial_year_start=form.get('financial_year_start'))
     return jsonify(form), 201
 
 
-def update_free_sms_fragment_limit_data(
-    service_id, free_sms_fragment_limit, financial_year_start
-):
+def update_free_sms_fragment_limit_data(service_id, free_sms_fragment_limit, financial_year_start):
     current_year = get_current_calendar_year_start_year()
     if not financial_year_start:
         financial_year_start = current_year
 
     dao_create_or_update_annual_billing_for_year(
-        service_id, free_sms_fragment_limit, financial_year_start
+        service_id,
+        free_sms_fragment_limit,
+        financial_year_start
     )
     # if we're trying to update historical data, don't touch other rows.
     # Otherwise, make sure that future years will get the new updated value.
     if financial_year_start >= current_year:
         dao_update_annual_billing_for_future_years(
-            service_id, free_sms_fragment_limit, financial_year_start
+            service_id,
+            free_sms_fragment_limit,
+            financial_year_start
         )

app/celery/nightly_tasks.py

@@ -25,15 +25,14 @@ from app.dao.notifications_dao import (
 from app.dao.service_data_retention_dao import (
     fetch_service_data_retention_for_all_services_by_notification_type,
 )
-from app.enums import NotificationType
-from app.models import FactProcessingTime
-from app.utils import get_midnight_in_utc, utc_now
+from app.models import EMAIL_TYPE, SMS_TYPE, FactProcessingTime
+from app.utils import get_midnight_in_utc
 
 
 @notify_celery.task(name="remove_sms_email_jobs")
 @cronitor("remove_sms_email_jobs")
 def remove_sms_email_csv_files():
-    _remove_csv_files([NotificationType.EMAIL, NotificationType.SMS])
+    _remove_csv_files([EMAIL_TYPE, SMS_TYPE])
 
 
 def _remove_csv_files(job_types):
@@ -46,7 +45,7 @@ def _remove_csv_files(job_types):
 
 @notify_celery.task(name="cleanup-unfinished-jobs")
 def cleanup_unfinished_jobs():
-    now = utc_now()
+    now = datetime.utcnow()
     jobs = dao_get_unfinished_jobs()
     for job in jobs:
         # The query already checks that the processing_finished time is null, so here we are saying
@@ -59,107 +58,88 @@ def cleanup_unfinished_jobs():
 
 @notify_celery.task(name="delete-notifications-older-than-retention")
 def delete_notifications_older_than_retention():
-    delete_email_notifications_older_than_retention.apply_async(
-        queue=QueueNames.REPORTING
-    )
-    delete_sms_notifications_older_than_retention.apply_async(
-        queue=QueueNames.REPORTING
-    )
+    delete_email_notifications_older_than_retention.apply_async(queue=QueueNames.REPORTING)
+    delete_sms_notifications_older_than_retention.apply_async(queue=QueueNames.REPORTING)
 
 
 @notify_celery.task(name="delete-sms-notifications")
 @cronitor("delete-sms-notifications")
 def delete_sms_notifications_older_than_retention():
-    _delete_notifications_older_than_retention_by_type(NotificationType.SMS)
+    _delete_notifications_older_than_retention_by_type('sms')
 
 
 @notify_celery.task(name="delete-email-notifications")
 @cronitor("delete-email-notifications")
 def delete_email_notifications_older_than_retention():
-    _delete_notifications_older_than_retention_by_type(NotificationType.EMAIL)
+    _delete_notifications_older_than_retention_by_type('email')
 
 
 def _delete_notifications_older_than_retention_by_type(notification_type):
-    flexible_data_retention = (
-        fetch_service_data_retention_for_all_services_by_notification_type(
-            notification_type
-        )
-    )
+    flexible_data_retention = fetch_service_data_retention_for_all_services_by_notification_type(notification_type)
 
     for f in flexible_data_retention:
-        day_to_delete_backwards_from = get_midnight_in_utc(
-            utc_now()
-        ).date() - timedelta(days=f.days_of_retention)
+        day_to_delete_backwards_from = get_midnight_in_utc(datetime.utcnow()).date() \
+                                       - timedelta(days=f.days_of_retention)
 
-        delete_notifications_for_service_and_type.apply_async(
-            queue=QueueNames.REPORTING,
-            kwargs={
-                "service_id": f.service_id,
-                "notification_type": notification_type,
-                "datetime_to_delete_before": day_to_delete_backwards_from,
-            },
-        )
+        delete_notifications_for_service_and_type.apply_async(queue=QueueNames.REPORTING, kwargs={
+            'service_id': f.service_id,
+            'notification_type': notification_type,
+            'datetime_to_delete_before': day_to_delete_backwards_from
+        })
 
-    seven_days_ago = get_midnight_in_utc(utc_now()).date() - timedelta(days=7)
+    seven_days_ago = get_midnight_in_utc(datetime.utcnow()).date() - timedelta(days=7)
 
     service_ids_with_data_retention = {x.service_id for x in flexible_data_retention}
 
     # get a list of all service ids that we'll need to delete for. Typically that might only be 5% of services.
     # This query takes a couple of mins to run.
-    service_ids_that_have_sent_notifications_recently = (
-        get_service_ids_with_notifications_before(notification_type, seven_days_ago)
+    service_ids_that_have_sent_notifications_recently = get_service_ids_with_notifications_before(
+        notification_type,
+        seven_days_ago
     )
 
-    service_ids_to_purge = (
-        service_ids_that_have_sent_notifications_recently
-        - service_ids_with_data_retention
-    )
+    service_ids_to_purge = service_ids_that_have_sent_notifications_recently - service_ids_with_data_retention
 
     for service_id in service_ids_to_purge:
-        delete_notifications_for_service_and_type.apply_async(
-            queue=QueueNames.REPORTING,
-            kwargs={
-                "service_id": service_id,
-                "notification_type": notification_type,
-                "datetime_to_delete_before": seven_days_ago,
-            },
-        )
+        delete_notifications_for_service_and_type.apply_async(queue=QueueNames.REPORTING, kwargs={
+            'service_id': service_id,
+            'notification_type': notification_type,
+            'datetime_to_delete_before': seven_days_ago
+        })
 
     current_app.logger.info(
-        f"delete-notifications-older-than-retention: triggered subtasks for notification_type {notification_type}: "
-        f"{len(service_ids_with_data_retention)} services with flexible data retention, "
-        f"{len(service_ids_to_purge)} services without flexible data retention"
+        f'delete-notifications-older-than-retention: triggered subtasks for notification_type {notification_type}: '
+        f'{len(service_ids_with_data_retention)} services with flexible data retention, '
+        f'{len(service_ids_to_purge)} services without flexible data retention'
     )
 
 
-@notify_celery.task(name="delete-notifications-for-service-and-type")
-def delete_notifications_for_service_and_type(
-    service_id, notification_type, datetime_to_delete_before
-):
-    start = utc_now()
+@notify_celery.task(name='delete-notifications-for-service-and-type')
+def delete_notifications_for_service_and_type(service_id, notification_type, datetime_to_delete_before):
+    start = datetime.utcnow()
     num_deleted = move_notifications_to_notification_history(
         notification_type,
         service_id,
         datetime_to_delete_before,
     )
     if num_deleted:
-        end = utc_now()
+        end = datetime.utcnow()
         current_app.logger.info(
-            f"delete-notifications-for-service-and-type: "
-            f"service: {service_id}, "
-            f"notification_type: {notification_type}, "
-            f"count deleted: {num_deleted}, "
-            f"duration: {(end - start).seconds} seconds"
+            f'delete-notifications-for-service-and-type: '
+            f'service: {service_id}, '
+            f'notification_type: {notification_type}, '
+            f'count deleted: {num_deleted}, '
+            f'duration: {(end - start).seconds} seconds'
         )
 
 
-@notify_celery.task(name="timeout-sending-notifications")
-@cronitor("timeout-sending-notifications")
+@notify_celery.task(name='timeout-sending-notifications')
+@cronitor('timeout-sending-notifications')
 def timeout_notifications():
-    notifications = ["dummy value so len() > 0"]
+    notifications = ['dummy value so len() > 0']
 
-    cutoff_time = utc_now() - timedelta(
-        seconds=current_app.config.get("SENDING_NOTIFICATIONS_TIMEOUT_PERIOD")
+    cutoff_time = datetime.utcnow() - timedelta(
+        seconds=current_app.config.get('SENDING_NOTIFICATIONS_TIMEOUT_PERIOD')
     )
 
     while len(notifications) > 0:
@@ -169,21 +149,20 @@ def timeout_notifications():
             check_and_queue_callback_task(notification)
 
         current_app.logger.info(
-            "Timeout period reached for {} notifications, status has been updated.".format(
-                len(notifications)
-            )
-        )
+            "Timeout period reached for {} notifications, status has been updated.".format(len(notifications)))
 
 
 @notify_celery.task(name="delete-inbound-sms")
 @cronitor("delete-inbound-sms")
 def delete_inbound_sms():
     try:
-        start = utc_now()
+        start = datetime.utcnow()
         deleted = delete_inbound_sms_older_than_retention()
         current_app.logger.info(
             "Delete inbound sms job started {} finished {} deleted {} inbound sms notifications".format(
-                start, utc_now(), deleted
+                start,
+                datetime.utcnow(),
+                deleted
             )
         )
     except SQLAlchemyError:
@@ -191,13 +170,14 @@ def delete_inbound_sms():
         raise
 
 
-@notify_celery.task(name="save-daily-notification-processing-time")
+@notify_celery.task(name='save-daily-notification-processing-time')
 @cronitor("save-daily-notification-processing-time")
 def save_daily_notification_processing_time(local_date=None):
+
     # local_date is a string in the format of "YYYY-MM-DD"
     if local_date is None:
         # if a date is not provided, we run against yesterdays data
-        local_date = (utc_now() - timedelta(days=1)).date()
+        local_date = (datetime.utcnow() - timedelta(days=1)).date()
     else:
         local_date = datetime.strptime(local_date, "%Y-%m-%d").date()
 
@@ -208,6 +188,6 @@ def save_daily_notification_processing_time(local_date=None):
         FactProcessingTime(
             local_date=local_date,
             messages_total=result.messages_total,
-            messages_within_10_secs=result.messages_within_10_secs,
+            messages_within_10_secs=result.messages_within_10_secs
         )
     )

app/celery/process_ses_receipts_tasks.py

@@ -1,4 +1,4 @@
-from datetime import timedelta
+from datetime import datetime, timedelta
 
 import iso8601
 from celery.exceptions import Retry
@@ -20,14 +20,10 @@ from app.dao.service_callback_api_dao import (
     get_service_complaint_callback_api_for_service,
     get_service_delivery_status_callback_api_for_service,
 )
-from app.enums import CallbackType, NotificationStatus
-from app.models import Complaint
-from app.utils import utc_now
+from app.models import NOTIFICATION_PENDING, NOTIFICATION_SENDING, Complaint
 
 
-@notify_celery.task(
-    bind=True, name="process-ses-result", max_retries=5, default_retry_delay=300
-)
+@notify_celery.task(bind=True, name="process-ses-result", max_retries=5, default_retry_delay=300)
 def process_ses_results(self, response):
     try:
         ses_message = json.loads(response["Message"])
@@ -39,9 +35,9 @@ def process_ses_results(self, response):
         )
         bounce_message = None
 
-        if notification_type == "Bounce":
+        if notification_type == 'Bounce':
             bounce_message = determine_notification_bounce_type(ses_message)
-        elif notification_type == "Complaint":
+        elif notification_type == 'Complaint':
             _check_and_queue_complaint_callback_task(*handle_complaint(ses_message))
             return True
 
@@ -51,14 +47,10 @@ def process_ses_results(self, response):
         reference = ses_message["mail"]["messageId"]
 
         try:
-            notification = notifications_dao.dao_get_notification_by_reference(
-                reference
-            )
+            notification = notifications_dao.dao_get_notification_by_reference(reference)
         except NoResultFound:
-            message_time = iso8601.parse_date(ses_message["mail"]["timestamp"]).replace(
-                tzinfo=None
-            )
-            if utc_now() - message_time < timedelta(minutes=5):
+            message_time = iso8601.parse_date(ses_message["mail"]["timestamp"]).replace(tzinfo=None)
+            if datetime.utcnow() - message_time < timedelta(minutes=5):
                 current_app.logger.info(
                     f"Notification not found for reference: {reference}"
                     f"(while attempting update to {notification_status}). "
@@ -74,16 +66,12 @@ def process_ses_results(self, response):
             return
 
         if bounce_message:
-            current_app.logger.info(
-                f"SES bounce for notification ID {notification.id}: {bounce_message}"
-            )
+            current_app.logger.info(f"SES bounce for notification ID {notification.id}: {bounce_message}")
 
-        if notification.status not in {
-            NotificationStatus.SENDING,
-            NotificationStatus.PENDING,
-        }:
+        if notification.status not in {NOTIFICATION_SENDING, NOTIFICATION_PENDING}:
             notifications_dao._duplicate_update_warning(
-                notification, notification_status
+                notification,
+                notification_status
             )
             return
 
@@ -101,9 +89,7 @@ def process_ses_results(self, response):
             )
         else:
             current_app.logger.info(
-                "SES callback return status of {} for notification: {}".format(
-                    notification_status, notification.id
-                )
+                "SES callback return status of {} for notification: {}".format(notification_status, notification.id)
             )
 
         check_and_queue_callback_task(notification)
@@ -127,11 +113,7 @@ def determine_notification_bounce_type(ses_message):
         raise KeyError(f"Unhandled sns notification type {notification_type}")
 
     remove_emails_from_bounce(ses_message)
-    current_app.logger.info(
-        "SES bounce dict: {}".format(
-            json.dumps(ses_message).replace("{", "(").replace("}", ")")
-        )
-    )
+    current_app.logger.info("SES bounce dict: {}".format(json.dumps(ses_message).replace("{", "(").replace("}", ")")))
     if ses_message["bounce"]["bounceType"] == "Permanent":
         return "Permanent"
     return "Temporary"
@@ -139,9 +121,9 @@ def determine_notification_bounce_type(ses_message):
 
 def determine_notification_type(ses_message):
     notification_type = ses_message["notificationType"]
-    if notification_type not in ["Bounce", "Complaint", "Delivery"]:
+    if notification_type not in ["Bounce", "Complaint",   "Delivery"]:
         raise KeyError(f"Unhandled sns notification type {notification_type}")
-    if notification_type == "Bounce":
+    if notification_type == 'Bounce':
         return determine_notification_bounce_type(ses_message)
     return notification_type
 
@@ -171,22 +153,22 @@ def get_aws_responses(ses_message):
         "Permanent": {
             "message": "Hard bounced",
             "success": False,
-            "notification_status": NotificationStatus.PERMANENT_FAILURE,
+            "notification_status": "permanent-failure",
         },
         "Temporary": {
             "message": "Soft bounced",
             "success": False,
-            "notification_status": NotificationStatus.TEMPORARY_FAILURE,
+            "notification_status": "temporary-failure",
         },
         "Delivery": {
             "message": "Delivered",
             "success": True,
-            "notification_status": NotificationStatus.DELIVERED,
+            "notification_status": "delivered",
         },
         "Complaint": {
             "message": "Complaint",
             "success": True,
-            "notification_status": NotificationStatus.DELIVERED,
+            "notification_status": "delivered",
         },
     }[status]
 
@@ -198,29 +180,21 @@ def get_aws_responses(ses_message):
 def handle_complaint(ses_message):
     recipient_email = remove_emails_from_complaint(ses_message)[0]
     current_app.logger.info(
-        "Complaint from SES: \n{}".format(
-            json.dumps(ses_message).replace("{", "(").replace("}", ")")
-        )
+        "Complaint from SES: \n{}".format(json.dumps(ses_message).replace("{", "(").replace("}", ")"))
     )
     try:
         reference = ses_message["mail"]["messageId"]
     except KeyError as e:
-        current_app.logger.exception(
-            f"Complaint from SES failed to get reference from message with error: {e}"
-        )
+        current_app.logger.exception(f"Complaint from SES failed to get reference from message with error: {e}")
         return
     notification = dao_get_notification_history_by_reference(reference)
-    ses_complaint = ses_message.get(CallbackType.COMPLAINT, None)
+    ses_complaint = ses_message.get("complaint", None)
 
     complaint = Complaint(
         notification_id=notification.id,
         service_id=notification.service_id,
-        ses_feedback_id=(
-            ses_complaint.get("feedbackId", None) if ses_complaint else None
-        ),
-        complaint_type=(
-            ses_complaint.get("complaintFeedbackType", None) if ses_complaint else None
-        ),
+        ses_feedback_id=ses_complaint.get("feedbackId", None) if ses_complaint else None,
+        complaint_type=ses_complaint.get("complaintFeedbackType", None) if ses_complaint else None,
         complaint_date=ses_complaint.get("timestamp", None) if ses_complaint else None,
     )
     save_complaint(complaint)
@@ -242,19 +216,15 @@ def remove_emails_from_bounce(bounce_dict):
 
 def remove_emails_from_complaint(complaint_dict):
     remove_mail_headers(complaint_dict)
-    complaint_dict[CallbackType.COMPLAINT].pop("complainedRecipients")
+    complaint_dict["complaint"].pop("complainedRecipients")
     return complaint_dict["mail"].pop("destination")
 
 
 def check_and_queue_callback_task(notification):
     # queue callback task only if the service_callback_api exists
-    service_callback_api = get_service_delivery_status_callback_api_for_service(
-        service_id=notification.service_id
-    )
+    service_callback_api = get_service_delivery_status_callback_api_for_service(service_id=notification.service_id)
     if service_callback_api:
-        notification_data = create_delivery_status_callback_data(
-            notification, service_callback_api
-        )
+        notification_data = create_delivery_status_callback_data(notification, service_callback_api)
         send_delivery_status_to_service.apply_async(
             [str(notification.id), notification_data], queue=QueueNames.CALLBACKS
         )
@@ -262,13 +232,7 @@ def check_and_queue_callback_task(notification):
 
 def _check_and_queue_complaint_callback_task(complaint, notification, recipient):
     # queue callback task only if the service_callback_api exists
-    service_callback_api = get_service_complaint_callback_api_for_service(
-        service_id=notification.service_id
-    )
+    service_callback_api = get_service_complaint_callback_api_for_service(service_id=notification.service_id)
     if service_callback_api:
-        complaint_data = create_complaint_callback_data(
-            complaint, notification, service_callback_api, recipient
-        )
-        send_complaint_to_service.apply_async(
-            [complaint_data], queue=QueueNames.CALLBACKS
-        )
+        complaint_data = create_complaint_callback_data(complaint, notification, service_callback_api, recipient)
+        send_complaint_to_service.apply_async([complaint_data], queue=QueueNames.CALLBACKS)

app/celery/provider_tasks.py

@@ -1,12 +1,10 @@
-import json
-import os
-from datetime import timedelta
+from datetime import datetime, timedelta
+from time import time
 
-from botocore.exceptions import ClientError
 from flask import current_app
 from sqlalchemy.orm.exc import NoResultFound
 
-from app import aws_cloudwatch_client, notify_celery, redis_store
+from app import aws_cloudwatch_client, notify_celery
 from app.clients.email import EmailClientNonRetryableException
 from app.clients.email.aws_ses import AwsSesClientThrottlingSendRateException
 from app.clients.sms import SmsClientResponseException
@@ -17,21 +15,15 @@ from app.dao.notifications_dao import (
     update_notification_status_by_id,
 )
 from app.delivery import send_to_providers
-from app.enums import NotificationStatus
 from app.exceptions import NotificationTechnicalFailureException
-from app.utils import utc_now
-
-# This is the amount of time to wait after sending an sms message before we check the aws logs and look for delivery
-# receipts
-DELIVERY_RECEIPT_DELAY_IN_SECONDS = 30
-
-
-@notify_celery.task(
-    bind=True,
-    name="check_sms_delivery_receipt",
-    max_retries=48,
-    default_retry_delay=300,
+from app.models import (
+    NOTIFICATION_DELIVERED,
+    NOTIFICATION_FAILED,
+    NOTIFICATION_TECHNICAL_FAILURE,
 )
+
+
+@notify_celery.task(bind=True, name="check_sms_delivery_receipt", max_retries=48, default_retry_delay=300)
 def check_sms_delivery_receipt(self, message_id, notification_id, sent_at):
     """
     This is called after deliver_sms to check the status of the message. This uses the same number of
@@ -42,109 +34,44 @@ def check_sms_delivery_receipt(self, message_id, notification_id, sent_at):
     failure appears in the cloudwatch logs, so this should keep retrying until the log appears, or until
     we run out of retries.
     """
-    # TODO the localstack cloudwatch doesn't currently have our log groups.  Possibly create them with awslocal?
-    if aws_cloudwatch_client.is_localstack():
-        status = "success"
-        provider_response = "this is a fake successful localstack sms message"
-        carrier = "unknown"
-    else:
-        try:
-            status, provider_response, carrier = aws_cloudwatch_client.check_sms(
-                message_id, notification_id, sent_at
-            )
-        except NotificationTechnicalFailureException as ntfe:
-            provider_response = "Unable to find carrier response -- still looking"
-            status = "pending"
-            carrier = ""
-            update_notification_status_by_id(
-                notification_id,
-                status,
-                carrier=carrier,
-                provider_response=provider_response,
-            )
-            raise self.retry(exc=ntfe)
-        except ClientError as err:
-            # Probably a ThrottlingException but could be something else
-            error_code = err.response["Error"]["Code"]
-            provider_response = (
-                f"{error_code} while checking sms receipt -- still looking"
-            )
-            status = "pending"
-            carrier = ""
-            update_notification_status_by_id(
-                notification_id,
-                status,
-                carrier=carrier,
-                provider_response=provider_response,
-            )
-            raise self.retry(exc=err)
-
-    if status == "success":
-        status = NotificationStatus.DELIVERED
-    elif status == "failure":
-        status = NotificationStatus.FAILED
+    status, provider_response = aws_cloudwatch_client.check_sms(message_id, notification_id, sent_at)
+    if status == 'success':
+        status = NOTIFICATION_DELIVERED
+    elif status == 'failure':
+        status = NOTIFICATION_FAILED
     # if status is not success or failure the client raised an exception and this method will retry
 
-    if status == NotificationStatus.DELIVERED:
-        sanitize_successful_notification_by_id(
-            notification_id, carrier=carrier, provider_response=provider_response
-        )
-        current_app.logger.info(
-            f"Sanitized notification {notification_id} that was successfully delivered"
-        )
+    if status == NOTIFICATION_DELIVERED:
+        sanitize_successful_notification_by_id(notification_id)
+        current_app.logger.info(f"Sanitized notification {notification_id} that was successfully delivered")
     else:
-        update_notification_status_by_id(
-            notification_id,
-            status,
-            carrier=carrier,
-            provider_response=provider_response,
-        )
-        current_app.logger.info(
-            f"Updated notification {notification_id} with response '{provider_response}'"
-        )
+        update_notification_status_by_id(notification_id, status, provider_response=provider_response)
+        current_app.logger.info(f"Updated notification {notification_id} with response '{provider_response}'")
 
 
-@notify_celery.task(
-    bind=True, name="deliver_sms", max_retries=48, default_retry_delay=300
-)
+@notify_celery.task(bind=True, name="deliver_sms", max_retries=48, default_retry_delay=300)
 def deliver_sms(self, notification_id):
-    """Branch off to the final step in delivering the notification to sns and get delivery receipts."""
     try:
-        current_app.logger.info(
-            "Start sending SMS for notification id: {}".format(notification_id)
-        )
+        # Get the time we are doing the sending, to minimize the time period we need to check over for receipt
+        now = round(time() * 1000)
+        current_app.logger.info("Start sending SMS for notification id: {}".format(notification_id))
         notification = notifications_dao.get_notification_by_id(notification_id)
-        ansi_green = "\033[32m"
-        ansi_reset = "\033[0m"
-
         if not notification:
             raise NoResultFound()
-        if (
-            os.getenv("NOTIFY_ENVIRONMENT") == "development"
-            and "authentication code" in notification.content
-        ):
-            current_app.logger.warning(
-                ansi_green + f"AUTHENTICATION CODE: {notification.content}" + ansi_reset
-            )
-        # Code branches off to send_to_providers.py
         message_id = send_to_providers.send_sms_to_provider(notification)
         # We have to put it in UTC.  For other timezones, the delay
         # will be ignored and it will fire immediately (although this probably only affects developer testing)
-        my_eta = utc_now() + timedelta(seconds=DELIVERY_RECEIPT_DELAY_IN_SECONDS)
+        my_eta = datetime.utcnow() + timedelta(seconds=300)
         check_sms_delivery_receipt.apply_async(
-            [message_id, notification_id, notification.created_at],
+            [message_id, notification_id, now],
             eta=my_eta,
-            queue=QueueNames.CHECK_SMS,
+            queue=QueueNames.CHECK_SMS
         )
     except Exception as e:
-        update_notification_status_by_id(
-            notification_id,
-            NotificationStatus.TEMPORARY_FAILURE,
-        )
         if isinstance(e, SmsClientResponseException):
             current_app.logger.warning(
                 "SMS notification delivery for id: {} failed".format(notification_id),
-                exc_info=True,
+                exc_info=True
             )
         else:
             current_app.logger.exception(
@@ -157,40 +84,25 @@ def deliver_sms(self, notification_id):
             else:
                 self.retry(queue=QueueNames.RETRY)
         except self.MaxRetriesExceededError:
-            message = (
-                "RETRY FAILED: Max retries reached. The task send_sms_to_provider failed for notification {}. "
-                "Notification has been updated to technical-failure".format(
-                    notification_id
-                )
-            )
-            update_notification_status_by_id(
-                notification_id,
-                NotificationStatus.TECHNICAL_FAILURE,
-            )
+            message = "RETRY FAILED: Max retries reached. The task send_sms_to_provider failed for notification {}. " \
+                      "Notification has been updated to technical-failure".format(notification_id)
+            update_notification_status_by_id(notification_id, NOTIFICATION_TECHNICAL_FAILURE)
             raise NotificationTechnicalFailureException(message)
 
 
-@notify_celery.task(
-    bind=True, name="deliver_email", max_retries=48, default_retry_delay=300
-)
+@notify_celery.task(bind=True, name="deliver_email", max_retries=48, default_retry_delay=300)
 def deliver_email(self, notification_id):
     try:
-        current_app.logger.info(
-            "Start sending email for notification id: {}".format(notification_id)
-        )
+        current_app.logger.info("Start sending email for notification id: {}".format(notification_id))
         notification = notifications_dao.get_notification_by_id(notification_id)
-
         if not notification:
             raise NoResultFound()
-        personalisation = redis_store.get(f"email-personalisation-{notification_id}")
-
-        notification.personalisation = json.loads(personalisation)
         send_to_providers.send_email_to_provider(notification)
     except EmailClientNonRetryableException as e:
         current_app.logger.exception(
             f"Email notification {notification_id} failed: {e}"
         )
-        update_notification_status_by_id(notification_id, "technical-failure")
+        update_notification_status_by_id(notification_id, 'technical-failure')
     except Exception as e:
         try:
             if isinstance(e, AwsSesClientThrottlingSendRateException):
@@ -204,15 +116,8 @@ def deliver_email(self, notification_id):
 
             self.retry(queue=QueueNames.RETRY)
         except self.MaxRetriesExceededError:
-            message = (
-                "RETRY FAILED: Max retries reached. "
-                "The task send_email_to_provider failed for notification {}. "
-                "Notification has been updated to technical-failure".format(
-                    notification_id
-                )
-            )
-            update_notification_status_by_id(
-                notification_id,
-                NotificationStatus.TECHNICAL_FAILURE,
-            )
+            message = "RETRY FAILED: Max retries reached. " \
+                      "The task send_email_to_provider failed for notification {}. " \
+                      "Notification has been updated to technical-failure".format(notification_id)
+            update_notification_status_by_id(notification_id, NOTIFICATION_TECHNICAL_FAILURE)
             raise NotificationTechnicalFailureException(message)

app/celery/reporting_tasks.py

@@ -5,11 +5,13 @@ from flask import current_app
 from app import notify_celery
 from app.config import QueueNames
 from app.cronitor import cronitor
-from app.dao.fact_billing_dao import fetch_billing_data_for_day, update_fact_billing
+from app.dao.fact_billing_dao import (
+    fetch_billing_data_for_day,
+    update_fact_billing,
+)
 from app.dao.fact_notification_status_dao import update_fact_notification_status
 from app.dao.notifications_dao import get_service_ids_with_notifications_on_date
-from app.enums import NotificationType
-from app.utils import utc_now
+from app.models import EMAIL_TYPE, SMS_TYPE
 
 
 @notify_celery.task(name="create-nightly-billing")
@@ -18,7 +20,7 @@ def create_nightly_billing(day_start=None):
     # day_start is a datetime.date() object. e.g.
     # up to 4 days of data counting back from day_start is consolidated
     if day_start is None:
-        day_start = utc_now().date() - timedelta(days=1)
+        day_start = datetime.utcnow().date() - timedelta(days=1)
     else:
         # When calling the task its a string in the format of "YYYY-MM-DD"
         day_start = datetime.strptime(day_start, "%Y-%m-%d").date()
@@ -26,7 +28,8 @@ def create_nightly_billing(day_start=None):
         process_day = (day_start - timedelta(days=i)).isoformat()
 
         create_nightly_billing_for_day.apply_async(
-            kwargs={"process_day": process_day}, queue=QueueNames.REPORTING
+            kwargs={'process_day': process_day},
+            queue=QueueNames.REPORTING
         )
         current_app.logger.info(
             f"create-nightly-billing task: create-nightly-billing-for-day task created for {process_day}"
@@ -37,15 +40,15 @@ def create_nightly_billing(day_start=None):
 def create_nightly_billing_for_day(process_day):
     process_day = datetime.strptime(process_day, "%Y-%m-%d").date()
     current_app.logger.info(
-        f"create-nightly-billing-for-day task for {process_day}: started"
+        f'create-nightly-billing-for-day task for {process_day}: started'
     )
 
-    start = utc_now()
+    start = datetime.utcnow()
     transit_data = fetch_billing_data_for_day(process_day=process_day)
-    end = utc_now()
+    end = datetime.utcnow()
 
     current_app.logger.info(
-        f"create-nightly-billing-for-day task for {process_day}: data fetched in {(end - start).seconds} seconds"
+        f'create-nightly-billing-for-day task for {process_day}: data fetched in {(end - start).seconds} seconds'
     )
 
     for data in transit_data:
@@ -79,9 +82,9 @@ def create_nightly_notification_status():
         mean the aggregated results are temporarily incorrect.
     """
 
-    yesterday = utc_now().date() - timedelta(days=1)
+    yesterday = datetime.utcnow().date() - timedelta(days=1)
 
-    for notification_type in (NotificationType.SMS, NotificationType.EMAIL):
+    for notification_type in [SMS_TYPE, EMAIL_TYPE]:
         days = 4
 
         for i in range(days):
@@ -94,30 +97,28 @@ def create_nightly_notification_status():
             for service_id in relevant_service_ids:
                 create_nightly_notification_status_for_service_and_day.apply_async(
                     kwargs={
-                        "process_day": process_day.isoformat(),
-                        "notification_type": notification_type,
-                        "service_id": service_id,
+                        'process_day': process_day.isoformat(),
+                        'notification_type': notification_type,
+                        'service_id': service_id,
                     },
-                    queue=QueueNames.REPORTING,
+                    queue=QueueNames.REPORTING
                 )
 
 
 @notify_celery.task(name="create-nightly-notification-status-for-service-and-day")
-def create_nightly_notification_status_for_service_and_day(
-    process_day, service_id, notification_type
-):
+def create_nightly_notification_status_for_service_and_day(process_day, service_id, notification_type):
     process_day = datetime.strptime(process_day, "%Y-%m-%d").date()
 
-    start = utc_now()
+    start = datetime.utcnow()
     update_fact_notification_status(
         process_day=process_day,
         notification_type=notification_type,
-        service_id=service_id,
+        service_id=service_id
     )
 
-    end = utc_now()
+    end = datetime.utcnow()
     current_app.logger.info(
-        f"create-nightly-notification-status-for-service-and-day task update "
-        f"for {service_id}, {notification_type} for {process_day}: "
-        f"updated in {(end - start).seconds} seconds"
+        f'create-nightly-notification-status-for-service-and-day task update '
+        f'for {service_id}, {notification_type} for {process_day}: '
+        f'updated in {(end - start).seconds} seconds'
     )

app/celery/research_mode_tasks.py

@@ -6,7 +6,7 @@ from requests import HTTPError, request
 from app.celery.process_ses_receipts_tasks import process_ses_results
 from app.config import QueueNames
 from app.dao.notifications_dao import get_notification_by_id
-from app.enums import NotificationType
+from app.models import SMS_TYPE
 
 temp_fail = "2028675303"
 perm_fail = "2028675302"
@@ -21,7 +21,7 @@ def send_sms_response(provider, reference):
     body = sns_callback(reference)
     headers = {"Content-type": "application/json"}
 
-    make_request(NotificationType.SMS, provider, body, headers)
+    make_request(SMS_TYPE, provider, body, headers)
 
 
 def send_email_response(reference, to):
@@ -36,17 +36,22 @@ def send_email_response(reference, to):
 
 
 def make_request(notification_type, provider, data, headers):
-    api_call = "{}/notifications/{}/{}".format(
-        current_app.config["API_HOST_NAME"], notification_type, provider
-    )
+    api_call = "{}/notifications/{}/{}".format(current_app.config["API_HOST_NAME"], notification_type, provider)
 
     try:
-        response = request("POST", api_call, headers=headers, data=data, timeout=60)
+        response = request(
+            "POST",
+            api_call,
+            headers=headers,
+            data=data,
+            timeout=60
+        )
         response.raise_for_status()
     except HTTPError as e:
         current_app.logger.error(
             "API POST request on {} failed with status {}".format(
-                api_call, e.response.status_code
+                api_call,
+                e.response.status_code
             )
         )
         raise e
@@ -60,131 +65,151 @@ def sns_callback(notification_id):
 
     # This will only work if all notifications, including successful ones, are in the notifications table
     # If we decide to delete successful notifications, we will have to get this from notifications history
-    return json.dumps(
-        {
-            "CID": str(notification_id),
-            "status": notification.status,
-            # "deliverytime": notification.completed_at
-        }
-    )
+    return json.dumps({
+        "CID": str(notification_id),
+        "status": notification.status,
+        # "deliverytime": notification.completed_at
+    })
 
 
 def ses_notification_callback(reference):
     ses_message_body = {
-        "delivery": {
-            "processingTimeMillis": 2003,
-            "recipients": ["success@simulator.amazonses.com"],
-            "remoteMtaIp": "123.123.123.123",
-            "reportingMTA": "a7-32.smtp-out.us-west-2.amazonses.com",
-            "smtpResponse": "250 2.6.0 Message received",
-            "timestamp": "2017-11-17T12:14:03.646Z",
+        'delivery': {
+            'processingTimeMillis': 2003,
+            'recipients': ['success@simulator.amazonses.com'],
+            'remoteMtaIp': '123.123.123.123',
+            'reportingMTA': 'a7-32.smtp-out.us-west-2.amazonses.com',
+            'smtpResponse': '250 2.6.0 Message received',
+            'timestamp': '2017-11-17T12:14:03.646Z'
         },
-        "mail": {
-            "commonHeaders": {
-                "from": ["TEST <TEST@notify.works>"],
-                "subject": "lambda test",
-                "to": ["success@simulator.amazonses.com"],
+        'mail': {
+            'commonHeaders': {
+                'from': ['TEST <TEST@notify.works>'],
+                'subject': 'lambda test',
+                'to': ['success@simulator.amazonses.com']
             },
-            "destination": ["success@simulator.amazonses.com"],
-            "headers": [
-                {"name": "From", "value": "TEST <TEST@notify.works>"},
-                {"name": "To", "value": "success@simulator.amazonses.com"},
-                {"name": "Subject", "value": "lambda test"},
-                {"name": "MIME-Version", "value": "1.0"},
+            'destination': ['success@simulator.amazonses.com'],
+            'headers': [
                 {
-                    "name": "Content-Type",
-                    "value": 'multipart/alternative; boundary="----=_Part_617203_1627511946.1510920841645"',
+                    'name': 'From',
+                    'value': 'TEST <TEST@notify.works>'
                 },
+                {
+                    'name': 'To',
+                    'value': 'success@simulator.amazonses.com'
+                },
+                {
+                    'name': 'Subject',
+                    'value': 'lambda test'
+                },
+                {
+                    'name': 'MIME-Version',
+                    'value': '1.0'
+                },
+                {
+                    'name': 'Content-Type',
+                    'value': 'multipart/alternative; boundary="----=_Part_617203_1627511946.1510920841645"'
+                }
             ],
-            "headersTruncated": False,
-            "messageId": reference,
-            "sendingAccountId": "12341234",
-            "source": '"TEST" <TEST@notify.works>',
-            "sourceArn": "arn:aws:ses:us-west-2:12341234:identity/notify.works",
-            "sourceIp": "0.0.0.1",
-            "timestamp": "2017-11-17T12:14:01.643Z",
+            'headersTruncated': False,
+            'messageId': reference,
+            'sendingAccountId': '12341234',
+            'source': '"TEST" <TEST@notify.works>',
+            'sourceArn': 'arn:aws:ses:us-west-2:12341234:identity/notify.works',
+            'sourceIp': '0.0.0.1',
+            'timestamp': '2017-11-17T12:14:01.643Z'
         },
-        "notificationType": "Delivery",
+        'notificationType': 'Delivery'
     }
 
     return {
-        "Type": "Notification",
-        "MessageId": "8e83c020-1234-1234-1234-92a8ee9baa0a",
-        "TopicArn": "arn:aws:sns:us-west-2:12341234:ses_notifications",
-        "Subject": None,
-        "Message": json.dumps(ses_message_body),
-        "Timestamp": "2017-11-17T12:14:03.710Z",
-        "SignatureVersion": "1",
-        "Signature": "[REDACTED]",
-        "SigningCertUrl": "https://sns.us-west-2.amazonaws.com/SimpleNotificationService-[REDACTED].pem",
-        "UnsubscribeUrl": "https://sns.us-west-2.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=[REACTED]",
-        "MessageAttributes": {},
+        'Type': 'Notification',
+        'MessageId': '8e83c020-1234-1234-1234-92a8ee9baa0a',
+        'TopicArn': 'arn:aws:sns:us-west-2:12341234:ses_notifications',
+        'Subject': None,
+        'Message': json.dumps(ses_message_body),
+        'Timestamp': '2017-11-17T12:14:03.710Z',
+        'SignatureVersion': '1',
+        'Signature': '[REDACTED]',
+        'SigningCertUrl': 'https://sns.us-west-2.amazonaws.com/SimpleNotificationService-[REDACTED].pem',
+        'UnsubscribeUrl': 'https://sns.us-west-2.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=[REACTED]',
+        'MessageAttributes': {}
     }
 
 
 def ses_hard_bounce_callback(reference):
-    return _ses_bounce_callback(reference, "Permanent")
+    return _ses_bounce_callback(reference, 'Permanent')
 
 
 def ses_soft_bounce_callback(reference):
-    return _ses_bounce_callback(reference, "Temporary")
+    return _ses_bounce_callback(reference, 'Temporary')
 
 
 def _ses_bounce_callback(reference, bounce_type):
     ses_message_body = {
-        "bounce": {
-            "bounceSubType": "General",
-            "bounceType": bounce_type,
-            "bouncedRecipients": [
+        'bounce': {
+            'bounceSubType': 'General',
+            'bounceType': bounce_type,
+            'bouncedRecipients': [{
+                'action': 'failed',
+                'diagnosticCode': 'smtp; 550 5.1.1 user unknown',
+                'emailAddress': 'bounce@simulator.amazonses.com',
+                'status': '5.1.1'
+            }],
+            'feedbackId': '0102015fc9e676fb-12341234-1234-1234-1234-9301e86a4fa8-000000',
+            'remoteMtaIp': '123.123.123.123',
+            'reportingMTA': 'dsn; a7-31.smtp-out.us-west-2.amazonses.com',
+            'timestamp': '2017-11-17T12:14:05.131Z'
+        },
+        'mail': {
+            'commonHeaders': {
+                'from': ['TEST <TEST@notify.works>'],
+                'subject': 'ses callback test',
+                'to': ['bounce@simulator.amazonses.com']
+            },
+            'destination': ['bounce@simulator.amazonses.com'],
+            'headers': [
                 {
-                    "action": "failed",
-                    "diagnosticCode": "smtp; 550 5.1.1 user unknown",
-                    "emailAddress": "bounce@simulator.amazonses.com",
-                    "status": "5.1.1",
+                    'name': 'From',
+                    'value': 'TEST <TEST@notify.works>'
+                },
+                {
+                    'name': 'To',
+                    'value': 'bounce@simulator.amazonses.com'
+                },
+                {
+                    'name': 'Subject',
+                    'value': 'lambda test'
+                },
+                {
+                    'name': 'MIME-Version',
+                    'value': '1.0'
+                },
+                {
+                    'name': 'Content-Type',
+                    'value': 'multipart/alternative; boundary="----=_Part_596529_2039165601.1510920843367"'
                 }
             ],
-            "feedbackId": "0102015fc9e676fb-12341234-1234-1234-1234-9301e86a4fa8-000000",
-            "remoteMtaIp": "123.123.123.123",
-            "reportingMTA": "dsn; a7-31.smtp-out.us-west-2.amazonses.com",
-            "timestamp": "2017-11-17T12:14:05.131Z",
+            'headersTruncated': False,
+            'messageId': reference,
+            'sendingAccountId': '12341234',
+            'source': '"TEST" <TEST@notify.works>',
+            'sourceArn': 'arn:aws:ses:us-west-2:12341234:identity/notify.works',
+            'sourceIp': '0.0.0.1',
+            'timestamp': '2017-11-17T12:14:03.000Z'
         },
-        "mail": {
-            "commonHeaders": {
-                "from": ["TEST <TEST@notify.works>"],
-                "subject": "ses callback test",
-                "to": ["bounce@simulator.amazonses.com"],
-            },
-            "destination": ["bounce@simulator.amazonses.com"],
-            "headers": [
-                {"name": "From", "value": "TEST <TEST@notify.works>"},
-                {"name": "To", "value": "bounce@simulator.amazonses.com"},
-                {"name": "Subject", "value": "lambda test"},
-                {"name": "MIME-Version", "value": "1.0"},
-                {
-                    "name": "Content-Type",
-                    "value": 'multipart/alternative; boundary="----=_Part_596529_2039165601.1510920843367"',
-                },
-            ],
-            "headersTruncated": False,
-            "messageId": reference,
-            "sendingAccountId": "12341234",
-            "source": '"TEST" <TEST@notify.works>',
-            "sourceArn": "arn:aws:ses:us-west-2:12341234:identity/notify.works",
-            "sourceIp": "0.0.0.1",
-            "timestamp": "2017-11-17T12:14:03.000Z",
-        },
-        "notificationType": "Bounce",
+        'notificationType': 'Bounce'
     }
     return {
-        "Type": "Notification",
-        "MessageId": "36e67c28-1234-1234-1234-2ea0172aa4a7",
-        "TopicArn": "arn:aws:sns:us-west-2:12341234:ses_notifications",
-        "Subject": None,
-        "Message": json.dumps(ses_message_body),
-        "Timestamp": "2017-11-17T12:14:05.149Z",
-        "SignatureVersion": "1",
-        "Signature": "[REDACTED]",  # noqa
-        "SigningCertUrl": "https://sns.us-west-2.amazonaws.com/SimpleNotificationService-[REDACTED]].pem",
-        "UnsubscribeUrl": "https://sns.us-west-2.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=[REDACTED]]",
-        "MessageAttributes": {},
+        'Type': 'Notification',
+        'MessageId': '36e67c28-1234-1234-1234-2ea0172aa4a7',
+        'TopicArn': 'arn:aws:sns:us-west-2:12341234:ses_notifications',
+        'Subject': None,
+        'Message': json.dumps(ses_message_body),
+        'Timestamp': '2017-11-17T12:14:05.149Z',
+        'SignatureVersion': '1',
+        'Signature': '[REDACTED]',  # noqa
+        'SigningCertUrl': 'https://sns.us-west-2.amazonaws.com/SimpleNotificationService-[REDACTED]].pem',
+        'UnsubscribeUrl': 'https://sns.us-west-2.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=[REDACTED]]',
+        'MessageAttributes': {}
     }

app/celery/scheduled_tasks.py

@@ -1,6 +1,9 @@
-from datetime import timedelta
+from datetime import datetime, timedelta
 
 from flask import current_app
+from notifications_utils.clients.zendesk.zendesk_client import (
+    NotifySupportTicket,
+)
 from sqlalchemy import between
 from sqlalchemy.exc import SQLAlchemyError
 
@@ -15,7 +18,9 @@ from app.config import QueueNames
 from app.dao.invited_org_user_dao import (
     delete_org_invitations_created_more_than_two_days_ago,
 )
-from app.dao.invited_user_dao import expire_invitations_created_more_than_two_days_ago
+from app.dao.invited_user_dao import (
+    delete_invitations_created_more_than_two_days_ago,
+)
 from app.dao.jobs_dao import (
     dao_set_scheduled_jobs_to_pending,
     dao_update_job,
@@ -23,18 +28,23 @@ from app.dao.jobs_dao import (
     find_missing_row_for_job,
 )
 from app.dao.notifications_dao import notifications_not_yet_sent
+from app.dao.provider_details_dao import (
+    dao_adjust_provider_priority_back_to_resting_points,
+)
 from app.dao.services_dao import (
     dao_find_services_sending_to_tv_numbers,
     dao_find_services_with_high_failure_rates,
 )
 from app.dao.users_dao import delete_codes_older_created_more_than_a_day_ago
-from app.enums import JobStatus, NotificationType
-from app.models import Job
+from app.models import (
+    EMAIL_TYPE,
+    JOB_STATUS_ERROR,
+    JOB_STATUS_IN_PROGRESS,
+    JOB_STATUS_PENDING,
+    SMS_TYPE,
+    Job,
+)
 from app.notifications.process_notifications import send_notification_to_queue
-from app.utils import utc_now
-from notifications_utils.clients.zendesk.zendesk_client import NotifySupportTicket
-
-MAX_NOTIFICATION_FAILS = 10000
 
 
 @notify_celery.task(name="run-scheduled-jobs")
@@ -42,9 +52,7 @@ def run_scheduled_jobs():
     try:
         for job in dao_set_scheduled_jobs_to_pending():
             process_job.apply_async([str(job.id)], queue=QueueNames.JOBS)
-            current_app.logger.info(
-                "Job ID {} added to process job queue".format(job.id)
-            )
+            current_app.logger.info("Job ID {} added to process job queue".format(job.id))
     except SQLAlchemyError:
         current_app.logger.exception("Failed to run scheduled jobs")
         raise
@@ -53,42 +61,36 @@ def run_scheduled_jobs():
 @notify_celery.task(name="delete-verify-codes")
 def delete_verify_codes():
     try:
-        start = utc_now()
+        start = datetime.utcnow()
         deleted = delete_codes_older_created_more_than_a_day_ago()
         current_app.logger.info(
-            "Delete job started {} finished {} deleted {} verify codes".format(
-                start, utc_now(), deleted
-            )
+            "Delete job started {} finished {} deleted {} verify codes".format(start, datetime.utcnow(), deleted)
         )
     except SQLAlchemyError:
         current_app.logger.exception("Failed to delete verify codes")
         raise
 
 
-@notify_celery.task(name="expire-or-delete-invitations")
-def expire_or_delete_invitations():
+@notify_celery.task(name="delete-invitations")
+def delete_invitations():
     try:
-        start = utc_now()
-        expired_invites = expire_invitations_created_more_than_two_days_ago()
+        start = datetime.utcnow()
+        deleted_invites = delete_invitations_created_more_than_two_days_ago()
+        deleted_invites += delete_org_invitations_created_more_than_two_days_ago()
         current_app.logger.info(
-            f"Expire job started {start} finished {utc_now()} expired {expired_invites} invitations"
-        )
-    except SQLAlchemyError:
-        current_app.logger.exception("Failed to expire invitations")
-        raise
-
-    try:
-        start = utc_now()
-        deleted_invites = delete_org_invitations_created_more_than_two_days_ago()
-        current_app.logger.info(
-            f"Delete job started {start} finished {utc_now()} deleted {deleted_invites} invitations"
+            "Delete job started {} finished {} deleted {} invitations".format(start, datetime.utcnow(), deleted_invites)
         )
     except SQLAlchemyError:
         current_app.logger.exception("Failed to delete invitations")
         raise
 
 
-@notify_celery.task(name="check-job-status")
+@notify_celery.task(name='tend-providers-back-to-middle')
+def tend_providers_back_to_middle():
+    dao_adjust_provider_priority_back_to_resting_points()
+
+
+@notify_celery.task(name='check-job-status')
 def check_job_status():
     """
     every x minutes do this check
@@ -102,94 +104,81 @@ def check_job_status():
         update the job_status to 'error'
         process the rows in the csv that are missing (in another task) just do the check here.
     """
-    thirty_minutes_ago = utc_now() - timedelta(minutes=30)
-    thirty_five_minutes_ago = utc_now() - timedelta(minutes=35)
+    thirty_minutes_ago = datetime.utcnow() - timedelta(minutes=30)
+    thirty_five_minutes_ago = datetime.utcnow() - timedelta(minutes=35)
 
     incomplete_in_progress_jobs = Job.query.filter(
-        Job.job_status == JobStatus.IN_PROGRESS,
-        between(Job.processing_started, thirty_five_minutes_ago, thirty_minutes_ago),
+        Job.job_status == JOB_STATUS_IN_PROGRESS,
+        between(Job.processing_started, thirty_five_minutes_ago, thirty_minutes_ago)
     )
     incomplete_pending_jobs = Job.query.filter(
-        Job.job_status == JobStatus.PENDING,
+        Job.job_status == JOB_STATUS_PENDING,
         Job.scheduled_for.isnot(None),
-        between(Job.scheduled_for, thirty_five_minutes_ago, thirty_minutes_ago),
+        between(Job.scheduled_for, thirty_five_minutes_ago, thirty_minutes_ago)
     )
 
-    jobs_not_complete_after_30_minutes = (
-        incomplete_in_progress_jobs.union(incomplete_pending_jobs)
-        .order_by(Job.processing_started, Job.scheduled_for)
-        .all()
-    )
+    jobs_not_complete_after_30_minutes = incomplete_in_progress_jobs.union(
+        incomplete_pending_jobs
+    ).order_by(
+        Job.processing_started, Job.scheduled_for
+    ).all()
 
     # temporarily mark them as ERROR so that they don't get picked up by future check_job_status tasks
     # if they haven't been re-processed in time.
     job_ids = []
     for job in jobs_not_complete_after_30_minutes:
-        job.job_status = JobStatus.ERROR
+        job.job_status = JOB_STATUS_ERROR
         dao_update_job(job)
         job_ids.append(str(job.id))
 
     if job_ids:
         current_app.logger.info("Job(s) {} have not completed.".format(job_ids))
-        process_incomplete_jobs.apply_async([job_ids], queue=QueueNames.JOBS)
+        process_incomplete_jobs.apply_async(
+            [job_ids],
+            queue=QueueNames.JOBS
+        )
 
 
-@notify_celery.task(name="replay-created-notifications")
+@notify_celery.task(name='replay-created-notifications')
 def replay_created_notifications():
     # if the notification has not be send after 1 hour, then try to resend.
-    resend_created_notifications_older_than = 60 * 60
-    for notification_type in (NotificationType.EMAIL, NotificationType.SMS):
+    resend_created_notifications_older_than = (60 * 60)
+    for notification_type in (EMAIL_TYPE, SMS_TYPE):
         notifications_to_resend = notifications_not_yet_sent(
-            resend_created_notifications_older_than, notification_type
+            resend_created_notifications_older_than,
+            notification_type
         )
 
         if len(notifications_to_resend) > 0:
-            current_app.logger.info(
-                "Sending {} {} notifications "
-                "to the delivery queue because the notification "
-                "status was created.".format(
-                    len(notifications_to_resend), notification_type
-                )
-            )
+            current_app.logger.info("Sending {} {} notifications "
+                                    "to the delivery queue because the notification "
+                                    "status was created.".format(len(notifications_to_resend), notification_type))
 
         for n in notifications_to_resend:
-            send_notification_to_queue(notification=n)
+            send_notification_to_queue(notification=n, research_mode=n.service.research_mode)
 
 
-@notify_celery.task(name="check-for-missing-rows-in-completed-jobs")
+@notify_celery.task(name='check-for-missing-rows-in-completed-jobs')
 def check_for_missing_rows_in_completed_jobs():
     jobs = find_jobs_with_missing_rows()
     for job in jobs:
-        (
-            recipient_csv,
-            template,
-            sender_id,
-        ) = get_recipient_csv_and_template_and_sender_id(job)
+        recipient_csv, template, sender_id = get_recipient_csv_and_template_and_sender_id(job)
         missing_rows = find_missing_row_for_job(job.id, job.notification_count)
         for row_to_process in missing_rows:
             row = recipient_csv[row_to_process.missing_row]
             current_app.logger.info(
-                "Processing missing row: {} for job: {}".format(
-                    row_to_process.missing_row, job.id
-                )
-            )
+                "Processing missing row: {} for job: {}".format(row_to_process.missing_row, job.id))
             process_row(row, template, job, job.service, sender_id=sender_id)
 
 
-@notify_celery.task(
-    name="check-for-services-with-high-failure-rates-or-sending-to-tv-numbers"
-)
+@notify_celery.task(name='check-for-services-with-high-failure-rates-or-sending-to-tv-numbers')
 def check_for_services_with_high_failure_rates_or_sending_to_tv_numbers():
-    start_date = utc_now() - timedelta(days=1)
-    end_date = utc_now()
+    start_date = (datetime.utcnow() - timedelta(days=1))
+    end_date = datetime.utcnow()
     message = ""
 
-    services_with_failures = dao_find_services_with_high_failure_rates(
-        start_date=start_date, end_date=end_date
-    )
-    services_sending_to_tv_numbers = dao_find_services_sending_to_tv_numbers(
-        start_date=start_date, end_date=end_date
-    )
+    services_with_failures = dao_find_services_with_high_failure_rates(start_date=start_date, end_date=end_date)
+    services_sending_to_tv_numbers = dao_find_services_sending_to_tv_numbers(start_date=start_date, end_date=end_date)
 
     if services_with_failures:
         message += "{} service(s) have had high permanent-failure rates for sms messages in last 24 hours:\n".format(
@@ -197,19 +186,17 @@ def check_for_services_with_high_failure_rates_or_sending_to_tv_numbers():
         )
         for service in services_with_failures:
             service_dashboard = "{}/services/{}".format(
-                current_app.config["ADMIN_BASE_URL"],
+                current_app.config['ADMIN_BASE_URL'],
                 str(service.service_id),
             )
-            message += "service: {} failure rate: {},\n".format(
-                service_dashboard, service.permanent_failure_rate
-            )
+            message += "service: {} failure rate: {},\n".format(service_dashboard, service.permanent_failure_rate)
     elif services_sending_to_tv_numbers:
         message += "{} service(s) have sent over 500 sms messages to tv numbers in last 24 hours:\n".format(
             len(services_sending_to_tv_numbers)
         )
         for service in services_sending_to_tv_numbers:
             service_dashboard = "{}/services/{}".format(
-                current_app.config["ADMIN_BASE_URL"],
+                current_app.config['ADMIN_BASE_URL'],
                 str(service.service_id),
             )
             message += "service: {} count of sms to tv numbers: {},\n".format(
@@ -219,15 +206,13 @@ def check_for_services_with_high_failure_rates_or_sending_to_tv_numbers():
     if services_with_failures or services_sending_to_tv_numbers:
         current_app.logger.warning(message)
 
-        if current_app.config["NOTIFY_ENVIRONMENT"] in ["live", "production", "test"]:
-            message += (
-                "\nYou can find instructions for this ticket in our manual:\n"
-                "https://github.com/alphagov/notifications-manuals/wiki/Support-Runbook#Deal-with-services-with-high-failure-rates-or-sending-sms-to-tv-numbers"  # noqa
-            )
+        if current_app.config['NOTIFY_ENVIRONMENT'] in ['live', 'production', 'test']:
+            message += ("\nYou can find instructions for this ticket in our manual:\n"
+                        "https://github.com/alphagov/notifications-manuals/wiki/Support-Runbook#Deal-with-services-with-high-failure-rates-or-sending-sms-to-tv-numbers")  # noqa
             ticket = NotifySupportTicket(
                 subject=f"[{current_app.config['NOTIFY_ENVIRONMENT']}] High failure rates for sms spotted for services",
                 message=message,
                 ticket_type=NotifySupportTicket.TYPE_INCIDENT,
-                technical_ticket=True,
+                technical_ticket=True
             )
             zendesk_client.send_ticket_to_zendesk(ticket)

app/celery/service_callback_tasks.py

@@ -8,106 +8,102 @@ from app.config import QueueNames
 from app.utils import DATETIME_FORMAT
 
 
-@notify_celery.task(
-    bind=True, name="send-delivery-status", max_retries=5, default_retry_delay=300
-)
-def send_delivery_status_to_service(self, notification_id, encrypted_status_update):
+@notify_celery.task(bind=True, name="send-delivery-status", max_retries=5, default_retry_delay=300)
+def send_delivery_status_to_service(
+    self, notification_id, encrypted_status_update
+):
     status_update = encryption.decrypt(encrypted_status_update)
 
     data = {
         "id": str(notification_id),
-        "reference": status_update["notification_client_reference"],
-        "to": status_update["notification_to"],
-        "status": status_update["notification_status"],
-        "created_at": status_update["notification_created_at"],
-        "completed_at": status_update["notification_updated_at"],
-        "sent_at": status_update["notification_sent_at"],
-        "notification_type": status_update["notification_type"],
-        "template_id": status_update["template_id"],
-        "template_version": status_update["template_version"],
+        "reference": status_update['notification_client_reference'],
+        "to": status_update['notification_to'],
+        "status": status_update['notification_status'],
+        "created_at": status_update['notification_created_at'],
+        "completed_at": status_update['notification_updated_at'],
+        "sent_at": status_update['notification_sent_at'],
+        "notification_type": status_update['notification_type'],
+        "template_id": status_update['template_id'],
+        "template_version": status_update['template_version']
     }
 
     _send_data_to_service_callback_api(
         self,
         data,
-        status_update["service_callback_api_url"],
-        status_update["service_callback_api_bearer_token"],
-        "send_delivery_status_to_service",
+        status_update['service_callback_api_url'],
+        status_update['service_callback_api_bearer_token'],
+        'send_delivery_status_to_service'
     )
 
 
-@notify_celery.task(
-    bind=True, name="send-complaint", max_retries=5, default_retry_delay=300
-)
+@notify_celery.task(bind=True, name="send-complaint", max_retries=5, default_retry_delay=300)
 def send_complaint_to_service(self, complaint_data):
     complaint = encryption.decrypt(complaint_data)
 
     data = {
-        "notification_id": complaint["notification_id"],
-        "complaint_id": complaint["complaint_id"],
-        "reference": complaint["reference"],
-        "to": complaint["to"],
-        "complaint_date": complaint["complaint_date"],
+        "notification_id": complaint['notification_id'],
+        "complaint_id": complaint['complaint_id'],
+        "reference": complaint['reference'],
+        "to": complaint['to'],
+        "complaint_date": complaint['complaint_date']
     }
 
     _send_data_to_service_callback_api(
         self,
         data,
-        complaint["service_callback_api_url"],
-        complaint["service_callback_api_bearer_token"],
-        "send_complaint_to_service",
+        complaint['service_callback_api_url'],
+        complaint['service_callback_api_bearer_token'],
+        'send_complaint_to_service'
     )
 
 
-def _send_data_to_service_callback_api(
-    self, data, service_callback_url, token, function_name
-):
-    notification_id = (
-        data["notification_id"] if "notification_id" in data else data["id"]
-    )
+def _send_data_to_service_callback_api(self, data, service_callback_url, token, function_name):
+    notification_id = (data["notification_id"] if "notification_id" in data else data["id"])
     try:
         response = request(
             method="POST",
             url=service_callback_url,
             data=json.dumps(data),
             headers={
-                "Content-Type": "application/json",
-                "Authorization": "Bearer {}".format(token),
+                'Content-Type': 'application/json',
+                'Authorization': 'Bearer {}'.format(token)
             },
-            timeout=5,
-        )
-        current_app.logger.info(
-            "{} sending {} to {}, response {}".format(
-                function_name,
-                notification_id,
-                service_callback_url,
-                response.status_code,
-            )
+            timeout=5
         )
+        current_app.logger.info('{} sending {} to {}, response {}'.format(
+            function_name,
+            notification_id,
+            service_callback_url,
+            response.status_code
+        ))
         response.raise_for_status()
     except RequestException as e:
         current_app.logger.warning(
             "{} request failed for notification_id: {} and url: {}. exception: {}".format(
-                function_name, notification_id, service_callback_url, e
+                function_name,
+                notification_id,
+                service_callback_url,
+                e
             )
         )
-        if (
-            not isinstance(e, HTTPError)
-            or e.response.status_code >= 500
-            or e.response.status_code == 429
-        ):
+        if not isinstance(e, HTTPError) or e.response.status_code >= 500 or e.response.status_code == 429:
             try:
                 self.retry(queue=QueueNames.CALLBACKS_RETRY)
             except self.MaxRetriesExceededError:
                 current_app.logger.warning(
                     "Retry: {} has retried the max num of times for callback url {} and notification_id: {}".format(
-                        function_name, service_callback_url, notification_id
+                        function_name,
+                        service_callback_url,
+                        notification_id
                     )
                 )
         else:
             current_app.logger.warning(
                 "{} callback is not being retried for notification_id: {} and url: {}. exception: {}".format(
-                    function_name, notification_id, service_callback_url, e
+                    function_name,
+                    notification_id,
+                    service_callback_url,
+                    e
                 )
             )
 
@@ -120,16 +116,9 @@ def create_delivery_status_callback_data(notification, service_callback_api):
         "notification_status": notification.status,
         "notification_provider_response": notification.provider_response,  # TODO do we test for provider_response?
         "notification_created_at": notification.created_at.strftime(DATETIME_FORMAT),
-        "notification_updated_at": (
-            notification.updated_at.strftime(DATETIME_FORMAT)
-            if notification.updated_at
-            else None
-        ),
-        "notification_sent_at": (
-            notification.sent_at.strftime(DATETIME_FORMAT)
-            if notification.sent_at
-            else None
-        ),
+        "notification_updated_at":
+            notification.updated_at.strftime(DATETIME_FORMAT) if notification.updated_at else None,
+        "notification_sent_at": notification.sent_at.strftime(DATETIME_FORMAT) if notification.sent_at else None,
         "notification_type": notification.notification_type,
         "service_callback_api_url": service_callback_api.url,
         "service_callback_api_bearer_token": service_callback_api.bearer_token,
@@ -139,9 +128,7 @@ def create_delivery_status_callback_data(notification, service_callback_api):
     return encryption.encrypt(data)
 
 
-def create_complaint_callback_data(
-    complaint, notification, service_callback_api, recipient
-):
+def create_complaint_callback_data(complaint, notification, service_callback_api, recipient):
     data = {
         "complaint_id": str(complaint.id),
         "notification_id": str(notification.id),

app/celery/tasks.py

@@ -1,6 +1,8 @@
 import json
+from datetime import datetime
 
 from flask import current_app
+from notifications_utils.recipients import RecipientCSV
 from requests import HTTPError, RequestException, request
 from sqlalchemy.exc import IntegrityError, SQLAlchemyError
 
@@ -18,70 +20,62 @@ from app.dao.service_email_reply_to_dao import dao_get_reply_to_by_id
 from app.dao.service_inbound_api_dao import get_service_inbound_api_for_service
 from app.dao.service_sms_sender_dao import dao_get_service_sms_senders_by_id
 from app.dao.templates_dao import dao_get_template_by_id
-from app.enums import JobStatus, KeyType, NotificationType
-from app.errors import TotalRequestsError
+from app.models import (
+    EMAIL_TYPE,
+    JOB_STATUS_CANCELLED,
+    JOB_STATUS_FINISHED,
+    JOB_STATUS_IN_PROGRESS,
+    JOB_STATUS_PENDING,
+    KEY_TYPE_NORMAL,
+    SMS_TYPE,
+)
 from app.notifications.process_notifications import persist_notification
-from app.notifications.validators import check_service_over_total_message_limit
+from app.notifications.validators import check_service_over_daily_message_limit
 from app.serialised_models import SerialisedService, SerialisedTemplate
 from app.service.utils import service_allowed_to_send_to
-from app.utils import DATETIME_FORMAT, hilite, utc_now
-from notifications_utils.recipients import RecipientCSV
+from app.utils import DATETIME_FORMAT
+from app.v2.errors import TooManyRequestsError
 
 
 @notify_celery.task(name="process-job")
 def process_job(job_id, sender_id=None):
-    """Update job status, get csv data from s3, and begin processing csv rows."""
-    start = utc_now()
+    start = datetime.utcnow()
     job = dao_get_job_by_id(job_id)
-    current_app.logger.info(
-        "Starting process-job task for job id {} with status: {}".format(
-            job_id, job.job_status
-        )
-    )
+    current_app.logger.info("Starting process-job task for job id {} with status: {}".format(job_id, job.job_status))
 
-    if job.job_status != JobStatus.PENDING:
+    if job.job_status != JOB_STATUS_PENDING:
         return
 
     service = job.service
 
-    job.job_status = JobStatus.IN_PROGRESS
+    job.job_status = JOB_STATUS_IN_PROGRESS
     job.processing_started = start
     dao_update_job(job)
 
     if not service.active:
-        job.job_status = JobStatus.CANCELLED
+        job.job_status = JOB_STATUS_CANCELLED
         dao_update_job(job)
         current_app.logger.warning(
-            "Job {} has been cancelled, service {} is inactive".format(
-                job_id, service.id
-            )
-        )
+            "Job {} has been cancelled, service {} is inactive".format(job_id, service.id))
         return
 
-    if __total_sending_limits_for_job_exceeded(service, job, job_id):
+    if __sending_limits_for_job_exceeded(service, job, job_id):
         return
 
-    recipient_csv, template, sender_id = get_recipient_csv_and_template_and_sender_id(
-        job
-    )
+    recipient_csv, template, sender_id = get_recipient_csv_and_template_and_sender_id(job)
 
-    current_app.logger.info(
-        "Starting job {} processing {} notifications".format(
-            job_id, job.notification_count
-        )
-    )
+    current_app.logger.info("Starting job {} processing {} notifications".format(job_id, job.notification_count))
 
     for row in recipient_csv.get_rows():
         process_row(row, template, job, service, sender_id=sender_id)
 
-    # End point/Exit point for message send flow.
     job_complete(job, start=start)
 
 
 def job_complete(job, resumed=False, start=None):
-    job.job_status = JobStatus.FINISHED
+    job.job_status = JOB_STATUS_FINISHED
 
-    finished = utc_now()
+    finished = datetime.utcnow()
     job.processing_finished = finished
     dao_update_job(job)
 
@@ -91,9 +85,7 @@ def job_complete(job, resumed=False, start=None):
         )
     else:
         current_app.logger.info(
-            "Job {} created at {} started at {} finished at {}".format(
-                job.id, job.created_at, start, finished
-            )
+            "Job {} created at {} started at {} finished at {}".format(job.id, job.created_at, start, finished)
         )
 
 
@@ -101,40 +93,35 @@ def get_recipient_csv_and_template_and_sender_id(job):
     db_template = dao_get_template_by_id(job.template_id, job.template_version)
     template = db_template._as_utils_template()
 
-    contents, meta_data = s3.get_job_and_metadata_from_s3(
-        service_id=str(job.service_id), job_id=str(job.id)
-    )
+    contents, meta_data = s3.get_job_and_metadata_from_s3(service_id=str(job.service_id), job_id=str(job.id))
     recipient_csv = RecipientCSV(contents, template=template)
 
     return recipient_csv, template, meta_data.get("sender_id")
 
 
 def process_row(row, template, job, service, sender_id=None):
-    """Branch off based on notification type, sms or email."""
     template_type = template.template_type
-    encrypted = encryption.encrypt(
-        {
-            "template": str(template.id),
-            "template_version": job.template_version,
-            "job": str(job.id),
-            "to": row.recipient,
-            "row_number": row.index,
-            "personalisation": dict(row.personalisation),
-        }
-    )
+    encrypted = encryption.encrypt({
+        'template': str(template.id),
+        'template_version': job.template_version,
+        'job': str(job.id),
+        'to': row.recipient,
+        'row_number': row.index,
+        'personalisation': dict(row.personalisation)
+    })
 
-    # Both save_sms and save_email have the same general
-    # persist logic.
-    send_fns = {NotificationType.SMS: save_sms, NotificationType.EMAIL: save_email}
+    send_fns = {
+        SMS_TYPE: save_sms,
+        EMAIL_TYPE: save_email
+    }
 
     send_fn = send_fns[template_type]
 
     task_kwargs = {}
     if sender_id:
-        task_kwargs["sender_id"] = sender_id
+        task_kwargs['sender_id'] = sender_id
 
     notification_id = create_uuid()
-    # Kick-off persisting notification in save_sms/save_email.
     send_fn.apply_async(
         (
             str(service.id),
@@ -142,123 +129,100 @@ def process_row(row, template, job, service, sender_id=None):
             encrypted,
         ),
         task_kwargs,
-        queue=QueueNames.DATABASE,
+        queue=QueueNames.DATABASE if not service.research_mode else QueueNames.RESEARCH_MODE
     )
     return notification_id
 
 
-def __total_sending_limits_for_job_exceeded(service, job, job_id):
+def __sending_limits_for_job_exceeded(service, job, job_id):
     try:
-        total_sent = check_service_over_total_message_limit(KeyType.NORMAL, service)
-        if total_sent + job.notification_count > service.total_message_limit:
-            raise TotalRequestsError(service.total_message_limit)
+        total_sent = check_service_over_daily_message_limit(KEY_TYPE_NORMAL, service)
+        if total_sent + job.notification_count > service.message_limit:
+            raise TooManyRequestsError(service.message_limit)
         else:
             return False
-    except TotalRequestsError:
-        job.job_status = "sending limits exceeded"
-        job.processing_finished = utc_now()
+    except TooManyRequestsError:
+        job.job_status = 'sending limits exceeded'
+        job.processing_finished = datetime.utcnow()
         dao_update_job(job)
-        current_app.logger.error(
-            "Job {} size {} error. Total sending limits {} exceeded".format(
-                job_id, job.notification_count, service.message_limit
-            )
+        current_app.logger.info(
+            "Job {} size {} error. Sending limits {} exceeded".format(
+                job_id, job.notification_count, service.message_limit)
         )
         return True
 
 
 @notify_celery.task(bind=True, name="save-sms", max_retries=5, default_retry_delay=300)
-def save_sms(self, service_id, notification_id, encrypted_notification, sender_id=None):
-    """Persist notification to db and place notification in queue to send to sns."""
+def save_sms(self,
+             service_id,
+             notification_id,
+             encrypted_notification,
+             sender_id=None):
     notification = encryption.decrypt(encrypted_notification)
-    # SerialisedService and SerialisedTemplate classes are
-    # used here to grab the same service and template from the cache
-    # to improve performance.
     service = SerialisedService.from_id(service_id)
     template = SerialisedTemplate.from_id_and_service_id(
-        notification["template"],
+        notification['template'],
         service_id=service.id,
-        version=notification["template_version"],
+        version=notification['template_version'],
     )
 
     if sender_id:
-        reply_to_text = dao_get_service_sms_senders_by_id(
-            service_id, sender_id
-        ).sms_sender
+        reply_to_text = dao_get_service_sms_senders_by_id(service_id, sender_id).sms_sender
     else:
         reply_to_text = template.reply_to_text
-    # Return False when trial mode services try sending notifications
-    # to non-team and non-simulated recipients.
-    if not service_allowed_to_send_to(notification["to"], service, KeyType.NORMAL):
-        current_app.logger.info(
-            hilite(
-                f"service not allowed to send for job_id {notification.get('job', None)}, aborting"
-            )
-        )
+
+    if not service_allowed_to_send_to(notification['to'], service, KEY_TYPE_NORMAL):
         current_app.logger.debug(
             "SMS {} failed as restricted service".format(notification_id)
         )
         return
 
     try:
-        job_id = notification.get("job", None)
-        created_by_id = None
-        if job_id:
-            job = dao_get_job_by_id(job_id)
-            created_by_id = job.created_by_id
-
         saved_notification = persist_notification(
-            template_id=notification["template"],
-            template_version=notification["template_version"],
-            recipient=notification["to"],
+            template_id=notification['template'],
+            template_version=notification['template_version'],
+            recipient=notification['to'],
             service=service,
-            personalisation=notification.get("personalisation"),
-            notification_type=NotificationType.SMS,
+            personalisation=notification.get('personalisation'),
+            notification_type=SMS_TYPE,
             api_key_id=None,
-            key_type=KeyType.NORMAL,
-            created_at=utc_now(),
-            created_by_id=created_by_id,
-            job_id=notification.get("job", None),
-            job_row_number=notification.get("row_number", None),
+            key_type=KEY_TYPE_NORMAL,
+            created_at=datetime.utcnow(),
+            job_id=notification.get('job', None),
+            job_row_number=notification.get('row_number', None),
             notification_id=notification_id,
-            reply_to_text=reply_to_text,
+            reply_to_text=reply_to_text
         )
 
-        # Kick off sns process in provider_tasks.py
-        sn = saved_notification
-        current_app.logger.info(
-            hilite(
-                f"Deliver sms for job_id: {sn.job_id} row_number: {sn.job_row_number}"
-            )
-        )
         provider_tasks.deliver_sms.apply_async(
-            [str(saved_notification.id)], queue=QueueNames.SEND_SMS
+            [str(saved_notification.id)],
+            queue=QueueNames.SEND_SMS if not service.research_mode else QueueNames.RESEARCH_MODE
         )
 
         current_app.logger.debug(
             "SMS {} created at {} for job {}".format(
                 saved_notification.id,
                 saved_notification.created_at,
-                notification.get("job", None),
-            )
+                notification.get('job', None))
         )
 
     except SQLAlchemyError as e:
         handle_exception(self, notification, notification_id, e)
 
 
-@notify_celery.task(
-    bind=True, name="save-email", max_retries=5, default_retry_delay=300
-)
-def save_email(
-    self, service_id, notification_id, encrypted_notification, sender_id=None
-):
+@notify_celery.task(bind=True, name="save-email", max_retries=5, default_retry_delay=300)
+def save_email(self,
+               service_id,
+               notification_id,
+               encrypted_notification,
+               sender_id=None):
     notification = encryption.decrypt(encrypted_notification)
 
     service = SerialisedService.from_id(service_id)
     template = SerialisedTemplate.from_id_and_service_id(
-        notification["template"],
+        notification['template'],
         service_id=service.id,
-        version=notification["template_version"],
+        version=notification['template_version'],
     )
 
     if sender_id:
@@ -266,143 +230,127 @@ def save_email(
     else:
         reply_to_text = template.reply_to_text
 
-    if not service_allowed_to_send_to(notification["to"], service, KeyType.NORMAL):
-        current_app.logger.info(
-            "Email {} failed as restricted service".format(notification_id)
-        )
+    if not service_allowed_to_send_to(notification['to'], service, KEY_TYPE_NORMAL):
+        current_app.logger.info("Email {} failed as restricted service".format(notification_id))
         return
 
     try:
         saved_notification = persist_notification(
-            template_id=notification["template"],
-            template_version=notification["template_version"],
-            recipient=notification["to"],
+            template_id=notification['template'],
+            template_version=notification['template_version'],
+            recipient=notification['to'],
             service=service,
-            personalisation=notification.get("personalisation"),
-            notification_type=NotificationType.EMAIL,
+            personalisation=notification.get('personalisation'),
+            notification_type=EMAIL_TYPE,
             api_key_id=None,
-            key_type=KeyType.NORMAL,
-            created_at=utc_now(),
-            job_id=notification.get("job", None),
-            job_row_number=notification.get("row_number", None),
+            key_type=KEY_TYPE_NORMAL,
+            created_at=datetime.utcnow(),
+            job_id=notification.get('job', None),
+            job_row_number=notification.get('row_number', None),
             notification_id=notification_id,
-            reply_to_text=reply_to_text,
+            reply_to_text=reply_to_text
         )
 
         provider_tasks.deliver_email.apply_async(
-            [str(saved_notification.id)], queue=QueueNames.SEND_EMAIL
+            [str(saved_notification.id)],
+            queue=QueueNames.SEND_EMAIL if not service.research_mode else QueueNames.RESEARCH_MODE
         )
 
-        current_app.logger.debug(
-            "Email {} created at {}".format(
-                saved_notification.id, saved_notification.created_at
-            )
-        )
+        current_app.logger.debug("Email {} created at {}".format(saved_notification.id, saved_notification.created_at))
     except SQLAlchemyError as e:
         handle_exception(self, notification, notification_id, e)
 
 
-@notify_celery.task(
-    bind=True, name="save-api-email", max_retries=5, default_retry_delay=300
-)
+@notify_celery.task(bind=True, name="save-api-email", max_retries=5, default_retry_delay=300)
 def save_api_email(self, encrypted_notification):
+
     save_api_email_or_sms(self, encrypted_notification)
 
 
-@notify_celery.task(
-    bind=True, name="save-api-sms", max_retries=5, default_retry_delay=300
-)
+@notify_celery.task(bind=True, name="save-api-sms", max_retries=5, default_retry_delay=300)
 def save_api_sms(self, encrypted_notification):
     save_api_email_or_sms(self, encrypted_notification)
 
 
 def save_api_email_or_sms(self, encrypted_notification):
     notification = encryption.decrypt(encrypted_notification)
-    service = SerialisedService.from_id(notification["service_id"])
-    q = (
-        QueueNames.SEND_EMAIL
-        if notification["notification_type"] == NotificationType.EMAIL
-        else QueueNames.SEND_SMS
-    )
-    provider_task = (
-        provider_tasks.deliver_email
-        if notification["notification_type"] == NotificationType.EMAIL
+    service = SerialisedService.from_id(notification['service_id'])
+    q = QueueNames.SEND_EMAIL if notification['notification_type'] == EMAIL_TYPE else QueueNames.SEND_SMS
+    provider_task = provider_tasks.deliver_email if notification['notification_type'] == EMAIL_TYPE \
         else provider_tasks.deliver_sms
-    )
     try:
+
         persist_notification(
             notification_id=notification["id"],
-            template_id=notification["template_id"],
-            template_version=notification["template_version"],
-            recipient=notification["to"],
+            template_id=notification['template_id'],
+            template_version=notification['template_version'],
+            recipient=notification['to'],
             service=service,
-            personalisation=notification.get("personalisation"),
-            notification_type=notification["notification_type"],
-            client_reference=notification["client_reference"],
-            api_key_id=notification.get("api_key_id"),
-            key_type=KeyType.NORMAL,
-            created_at=notification["created_at"],
-            reply_to_text=notification["reply_to_text"],
-            status=notification["status"],
-            document_download_count=notification["document_download_count"],
+            personalisation=notification.get('personalisation'),
+            notification_type=notification['notification_type'],
+            client_reference=notification['client_reference'],
+            api_key_id=notification.get('api_key_id'),
+            key_type=KEY_TYPE_NORMAL,
+            created_at=notification['created_at'],
+            reply_to_text=notification['reply_to_text'],
+            status=notification['status'],
+            document_download_count=notification['document_download_count']
         )
 
-        provider_task.apply_async([notification["id"]], queue=q)
+        q = q if not service.research_mode else QueueNames.RESEARCH_MODE
+        provider_task.apply_async(
+            [notification['id']],
+            queue=q
+        )
         current_app.logger.debug(
             f"{notification['notification_type']} {notification['id']} has been persisted and sent to delivery queue."
         )
     except IntegrityError:
-        current_app.logger.info(
-            f"{notification['notification_type']} {notification['id']} already exists."
-        )
+        current_app.logger.info(f"{notification['notification_type']} {notification['id']} already exists.")
 
     except SQLAlchemyError:
+
         try:
             self.retry(queue=QueueNames.RETRY)
         except self.MaxRetriesExceededError:
-            current_app.logger.error(
-                f"Max retry failed Failed to persist notification {notification['id']}"
-            )
+            current_app.logger.error(f"Max retry failed Failed to persist notification {notification['id']}")
 
 
 def handle_exception(task, notification, notification_id, exc):
     if not get_notification_by_id(notification_id):
-        retry_msg = "{task} notification for job {job} row number {row} and notification id {noti}".format(
+        retry_msg = '{task} notification for job {job} row number {row} and notification id {noti}'.format(
             task=task.__name__,
-            job=notification.get("job", None),
-            row=notification.get("row_number", None),
-            noti=notification_id,
+            job=notification.get('job', None),
+            row=notification.get('row_number', None),
+            noti=notification_id
         )
         # Sometimes, SQS plays the same message twice. We should be able to catch an IntegrityError, but it seems
         # SQLAlchemy is throwing a FlushError. So we check if the notification id already exists then do not
         # send to the retry queue.
         # This probably (hopefully) is not an issue with Redis as the celery backing store
-        current_app.logger.exception("Retry" + retry_msg)
+        current_app.logger.exception('Retry' + retry_msg)
         try:
             task.retry(queue=QueueNames.RETRY, exc=exc)
         except task.MaxRetriesExceededError:
-            current_app.logger.error("Max retry failed" + retry_msg)
+            current_app.logger.error('Max retry failed' + retry_msg)
 
 
-@notify_celery.task(
-    bind=True, name="send-inbound-sms", max_retries=5, default_retry_delay=300
-)
+@notify_celery.task(bind=True, name="send-inbound-sms", max_retries=5, default_retry_delay=300)
 def send_inbound_sms_to_service(self, inbound_sms_id, service_id):
     inbound_api = get_service_inbound_api_for_service(service_id=service_id)
     if not inbound_api:
         # No API data has been set for this service
         return
 
-    inbound_sms = dao_get_inbound_sms_by_id(
-        service_id=service_id, inbound_id=inbound_sms_id
-    )
+    inbound_sms = dao_get_inbound_sms_by_id(service_id=service_id,
+                                            inbound_id=inbound_sms_id)
     data = {
         "id": str(inbound_sms.id),
         # TODO: should we be validating and formatting the phone number here?
         "source_number": inbound_sms.user_number,
         "destination_number": inbound_sms.notify_number,
         "message": inbound_sms.content,
-        "date_received": inbound_sms.provider_date.strftime(DATETIME_FORMAT),
+        "date_received": inbound_sms.provider_date.strftime(DATETIME_FORMAT)
     }
 
     try:
@@ -411,49 +359,44 @@ def send_inbound_sms_to_service(self, inbound_sms_id, service_id):
             url=inbound_api.url,
             data=json.dumps(data),
             headers={
-                "Content-Type": "application/json",
-                "Authorization": "Bearer {}".format(inbound_api.bearer_token),
+                'Content-Type': 'application/json',
+                'Authorization': 'Bearer {}'.format(inbound_api.bearer_token)
             },
-            timeout=60,
+            timeout=60
         )
         current_app.logger.debug(
-            f"send_inbound_sms_to_service sending {inbound_sms_id} to {inbound_api.url}, "
-            + f"response {response.status_code}"
+            f"send_inbound_sms_to_service sending {inbound_sms_id} to {inbound_api.url}, " +
+            f"response {response.status_code}"
         )
         response.raise_for_status()
     except RequestException as e:
         current_app.logger.warning(
-            f"send_inbound_sms_to_service failed for service_id: {service_id} for inbound_sms_id: {inbound_sms_id} "
-            + f"and url: {inbound_api.url}. exception: {e}"
+            f"send_inbound_sms_to_service failed for service_id: {service_id} for inbound_sms_id: {inbound_sms_id} " +
+            f"and url: {inbound_api.url}. exception: {e}"
         )
         if not isinstance(e, HTTPError) or e.response.status_code >= 500:
             try:
                 self.retry(queue=QueueNames.RETRY)
             except self.MaxRetriesExceededError:
                 current_app.logger.error(
-                    "Retry: send_inbound_sms_to_service has retried the max number of"
-                    + f"times for service: {service_id} and inbound_sms {inbound_sms_id}"
+                    "Retry: send_inbound_sms_to_service has retried the max number of" +
+                    f"times for service: {service_id} and inbound_sms {inbound_sms_id}"
                 )
         else:
             current_app.logger.warning(
-                f"send_inbound_sms_to_service is not being retried for service_id: {service_id} for "
-                + f"inbound_sms id: {inbound_sms_id} and url: {inbound_api.url}. exception: {e}"
+                f"send_inbound_sms_to_service is not being retried for service_id: {service_id} for " +
+                f"inbound_sms id: {inbound_sms_id} and url: {inbound_api.url}. exception: {e}"
             )
 
 
-@notify_celery.task(name="regenerate-job-cache")
-def regenerate_job_cache():
-    s3.get_s3_files()
-
-
-@notify_celery.task(name="process-incomplete-jobs")
+@notify_celery.task(name='process-incomplete-jobs')
 def process_incomplete_jobs(job_ids):
     jobs = [dao_get_job_by_id(job_id) for job_id in job_ids]
 
     # reset the processing start time so that the check_job_status scheduled task doesn't pick this job up again
     for job in jobs:
-        job.job_status = JobStatus.IN_PROGRESS
-        job.processing_started = utc_now()
+        job.job_status = JOB_STATUS_IN_PROGRESS
+        job.processing_started = datetime.utcnow()
         dao_update_job(job)
 
     current_app.logger.info("Resuming Job(s) {}".format(job_ids))
@@ -471,13 +414,9 @@ def process_incomplete_job(job_id):
     else:
         resume_from_row = -1  # The first row in the csv with a number is row 0
 
-    current_app.logger.info(
-        "Resuming job {} from row {}".format(job_id, resume_from_row)
-    )
+    current_app.logger.info("Resuming job {} from row {}".format(job_id, resume_from_row))
 
-    recipient_csv, template, sender_id = get_recipient_csv_and_template_and_sender_id(
-        job
-    )
+    recipient_csv, template, sender_id = get_recipient_csv_and_template_and_sender_id(job)
 
     for row in recipient_csv.get_rows():
         if row.index > resume_from_row:

app/celery/test_key_tasks.py

@@ -1,190 +0,0 @@
-import json
-
-from flask import current_app
-from requests import HTTPError, request
-
-from app.celery.process_ses_receipts_tasks import process_ses_results
-from app.config import QueueNames
-from app.dao.notifications_dao import get_notification_by_id
-from app.enums import NotificationType
-
-temp_fail = "2028675303"
-perm_fail = "2028675302"
-delivered = "2028675309"
-
-delivered_email = "delivered@simulator.notify"
-perm_fail_email = "perm-fail@simulator.notify"
-temp_fail_email = "temp-fail@simulator.notify"
-
-
-def send_sms_response(provider, reference):
-    body = sns_callback(reference)
-    headers = {"Content-type": "application/json"}
-
-    make_request(NotificationType.SMS, provider, body, headers)
-
-
-def send_email_response(reference, to):
-    if to == perm_fail_email:
-        body = ses_hard_bounce_callback(reference)
-    elif to == temp_fail_email:
-        body = ses_soft_bounce_callback(reference)
-    else:
-        body = ses_notification_callback(reference)
-
-    process_ses_results.apply_async([body], queue=QueueNames.SEND_EMAIL)
-
-
-def make_request(notification_type, provider, data, headers):
-    api_call = "{}/notifications/{}/{}".format(
-        current_app.config["API_HOST_NAME"], notification_type, provider
-    )
-
-    try:
-        response = request("POST", api_call, headers=headers, data=data, timeout=60)
-        response.raise_for_status()
-    except HTTPError as e:
-        current_app.logger.error(
-            "API POST request on {} failed with status {}".format(
-                api_call, e.response.status_code
-            )
-        )
-        raise e
-    finally:
-        current_app.logger.info("Mocked provider callback request finished")
-    return response.json()
-
-
-def sns_callback(notification_id):
-    notification = get_notification_by_id(notification_id)
-
-    # This will only work if all notifications, including successful ones, are in the notifications table
-    # If we decide to delete successful notifications, we will have to get this from notifications history
-    return json.dumps(
-        {
-            "CID": str(notification_id),
-            "status": notification.status,
-            # "deliverytime": notification.completed_at
-        }
-    )
-
-
-def ses_notification_callback(reference):
-    ses_message_body = {
-        "delivery": {
-            "processingTimeMillis": 2003,
-            "recipients": ["success@simulator.amazonses.com"],
-            "remoteMtaIp": "123.123.123.123",
-            "reportingMTA": "a7-32.smtp-out.us-west-2.amazonses.com",
-            "smtpResponse": "250 2.6.0 Message received",
-            "timestamp": "2017-11-17T12:14:03.646Z",
-        },
-        "mail": {
-            "commonHeaders": {
-                "from": ["TEST <TEST@notify.works>"],
-                "subject": "lambda test",
-                "to": ["success@simulator.amazonses.com"],
-            },
-            "destination": ["success@simulator.amazonses.com"],
-            "headers": [
-                {"name": "From", "value": "TEST <TEST@notify.works>"},
-                {"name": "To", "value": "success@simulator.amazonses.com"},
-                {"name": "Subject", "value": "lambda test"},
-                {"name": "MIME-Version", "value": "1.0"},
-                {
-                    "name": "Content-Type",
-                    "value": 'multipart/alternative; boundary="----=_Part_617203_1627511946.1510920841645"',
-                },
-            ],
-            "headersTruncated": False,
-            "messageId": reference,
-            "sendingAccountId": "12341234",
-            "source": '"TEST" <TEST@notify.works>',
-            "sourceArn": "arn:aws:ses:us-west-2:12341234:identity/notify.works",
-            "sourceIp": "0.0.0.1",
-            "timestamp": "2017-11-17T12:14:01.643Z",
-        },
-        "notificationType": "Delivery",
-    }
-
-    return {
-        "Type": "Notification",
-        "MessageId": "8e83c020-1234-1234-1234-92a8ee9baa0a",
-        "TopicArn": "arn:aws:sns:us-west-2:12341234:ses_notifications",
-        "Subject": None,
-        "Message": json.dumps(ses_message_body),
-        "Timestamp": "2017-11-17T12:14:03.710Z",
-        "SignatureVersion": "1",
-        "Signature": "[REDACTED]",
-        "SigningCertUrl": "https://sns.us-west-2.amazonaws.com/SimpleNotificationService-[REDACTED].pem",
-        "UnsubscribeUrl": "https://sns.us-west-2.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=[REACTED]",
-        "MessageAttributes": {},
-    }
-
-
-def ses_hard_bounce_callback(reference):
-    return _ses_bounce_callback(reference, "Permanent")
-
-
-def ses_soft_bounce_callback(reference):
-    return _ses_bounce_callback(reference, "Temporary")
-
-
-def _ses_bounce_callback(reference, bounce_type):
-    ses_message_body = {
-        "bounce": {
-            "bounceSubType": "General",
-            "bounceType": bounce_type,
-            "bouncedRecipients": [
-                {
-                    "action": "failed",
-                    "diagnosticCode": "smtp; 550 5.1.1 user unknown",
-                    "emailAddress": "bounce@simulator.amazonses.com",
-                    "status": "5.1.1",
-                }
-            ],
-            "feedbackId": "0102015fc9e676fb-12341234-1234-1234-1234-9301e86a4fa8-000000",
-            "remoteMtaIp": "123.123.123.123",
-            "reportingMTA": "dsn; a7-31.smtp-out.us-west-2.amazonses.com",
-            "timestamp": "2017-11-17T12:14:05.131Z",
-        },
-        "mail": {
-            "commonHeaders": {
-                "from": ["TEST <TEST@notify.works>"],
-                "subject": "ses callback test",
-                "to": ["bounce@simulator.amazonses.com"],
-            },
-            "destination": ["bounce@simulator.amazonses.com"],
-            "headers": [
-                {"name": "From", "value": "TEST <TEST@notify.works>"},
-                {"name": "To", "value": "bounce@simulator.amazonses.com"},
-                {"name": "Subject", "value": "lambda test"},
-                {"name": "MIME-Version", "value": "1.0"},
-                {
-                    "name": "Content-Type",
-                    "value": 'multipart/alternative; boundary="----=_Part_596529_2039165601.1510920843367"',
-                },
-            ],
-            "headersTruncated": False,
-            "messageId": reference,
-            "sendingAccountId": "12341234",
-            "source": '"TEST" <TEST@notify.works>',
-            "sourceArn": "arn:aws:ses:us-west-2:12341234:identity/notify.works",
-            "sourceIp": "0.0.0.1",
-            "timestamp": "2017-11-17T12:14:03.000Z",
-        },
-        "notificationType": "Bounce",
-    }
-    return {
-        "Type": "Notification",
-        "MessageId": "36e67c28-1234-1234-1234-2ea0172aa4a7",
-        "TopicArn": "arn:aws:sns:us-west-2:12341234:ses_notifications",
-        "Subject": None,
-        "Message": json.dumps(ses_message_body),
-        "Timestamp": "2017-11-17T12:14:05.149Z",
-        "SignatureVersion": "1",
-        "Signature": "[REDACTED]",  # noqa
-        "SigningCertUrl": "https://sns.us-west-2.amazonaws.com/SimpleNotificationService-[REDACTED]].pem",
-        "UnsubscribeUrl": "https://sns.us-west-2.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=[REDACTED]]",
-        "MessageAttributes": {},
-    }

app/clients/__init__.py

@@ -1,37 +1,20 @@
-from abc import abstractmethod
-from typing import Protocol
-
-from botocore.config import Config
-
-from app.enums import NotificationType
-
-AWS_CLIENT_CONFIG = Config(
-    # This config is required to enable S3 to connect to FIPS-enabled
-    # endpoints.  See https://aws.amazon.com/compliance/fips/ for more
-    # information.
-    s3={
-        "addressing_style": "virtual",
-    },
-    use_fips_endpoint=True,
-)
-
-
 class ClientException(Exception):
-    """
+    '''
     Base Exceptions for sending notifications that fail
-    """
-
+    '''
     pass
 
 
-class Client(Protocol):
-    """
+class Client(object):
+    '''
     Base client for sending notifications.
-    """
+    '''
+    pass
 
-    @abstractmethod
-    def init_app(self, current_app, *args, **kwargs):
-        raise NotImplementedError("TODO: Need to implement.")
+
+STATISTICS_REQUESTED = 'requested'
+STATISTICS_DELIVERED = 'delivered'
+STATISTICS_FAILURE = 'failure'
 
 
 class NotificationProviderClients(object):
@@ -52,13 +35,10 @@ class NotificationProviderClients(object):
         return self.email_clients.get(name)
 
     def get_client_by_name_and_type(self, name, notification_type):
-        assert notification_type in {
-            NotificationType.EMAIL,
-            NotificationType.SMS,
-        }  # nosec B101
+        assert notification_type in ['email', 'sms']  # nosec B101
 
-        if notification_type == NotificationType.EMAIL:
+        if notification_type == 'email':
             return self.get_email_client(name)
 
-        if notification_type == NotificationType.SMS:
+        if notification_type == 'sms':
             return self.get_sms_client(name)

app/clients/cloudwatch/aws_cloudwatch.py

@@ -1,15 +1,11 @@
 import json
-import os
 import re
-from datetime import timedelta
+import time
 
 from boto3 import client
-from flask import current_app
 
-from app.clients import AWS_CLIENT_CONFIG, Client
+from app.clients import Client
 from app.cloudfoundry_config import cloud_config
-from app.exceptions import NotificationTechnicalFailureException
-from app.utils import hilite, utc_now
 
 
 class AwsCloudwatchClient(Client):
@@ -18,150 +14,76 @@ class AwsCloudwatchClient(Client):
     """
 
     def init_app(self, current_app, *args, **kwargs):
-        if os.getenv("LOCALSTACK_ENDPOINT_URL"):
-            self._client = client(
-                "logs",
-                region_name=cloud_config.sns_region,
-                aws_access_key_id=cloud_config.sns_access_key,
-                aws_secret_access_key=cloud_config.sns_secret_key,
-                config=AWS_CLIENT_CONFIG,
-                endpoint_url=os.getenv("LOCALSTACK_ENDPOINT_URL"),
-            )
-            self._is_localstack = True
-        else:
-            self._client = client(
-                "logs",
-                region_name=cloud_config.sns_region,
-                aws_access_key_id=cloud_config.sns_access_key,
-                aws_secret_access_key=cloud_config.sns_secret_key,
-                config=AWS_CLIENT_CONFIG,
-            )
-            self._is_localstack = False
-
+        self._client = client(
+            "logs",
+            region_name=cloud_config.sns_region,
+            aws_access_key_id=cloud_config.sns_access_key,
+            aws_secret_access_key=cloud_config.sns_secret_key
+        )
         super(Client, self).__init__(*args, **kwargs)
         self.current_app = current_app
         self._valid_sender_regex = re.compile(r"^\+?\d{5,14}$")
 
     @property
     def name(self):
-        return "cloudwatch"
-
-    def is_localstack(self):
-        return self._is_localstack
+        return 'cloudwatch'
 
     def _get_log(self, my_filter, log_group_name, sent_at):
+
         # Check all cloudwatch logs from the time the notification was sent (currently 5 minutes previously) until now
-        now = utc_now()
+        now = round(time.time() * 1000)
         beginning = sent_at
         next_token = None
         all_log_events = []
-        current_app.logger.info(f"START TIME {beginning} END TIME {now}")
-        # There has been a change somewhere and the time range we were previously using has become too
-        # narrow or wrong in some way, so events can't be found.  For the time being, adjust by adding
-        # a buffer on each side of 12 hours.
-        TWELVE_HOURS = 12 * 60 * 60 * 1000
         while True:
             if next_token:
                 response = self._client.filter_log_events(
                     logGroupName=log_group_name,
                     filterPattern=my_filter,
                     nextToken=next_token,
-                    startTime=int(beginning.timestamp() * 1000) - TWELVE_HOURS,
-                    endTime=int(now.timestamp() * 1000) + TWELVE_HOURS,
+                    startTime=beginning,
+                    endTime=now
                 )
             else:
                 response = self._client.filter_log_events(
                     logGroupName=log_group_name,
                     filterPattern=my_filter,
-                    startTime=int(beginning.timestamp() * 1000) - TWELVE_HOURS,
-                    endTime=int(now.timestamp() * 1000) + TWELVE_HOURS,
+                    startTime=beginning,
+                    endTime=now
                 )
-            log_events = response.get("events", [])
+            log_events = response.get('events', [])
             all_log_events.extend(log_events)
             if len(log_events) > 0:
                 # We found it
-
                 break
-            next_token = response.get("nextToken")
+            next_token = response.get('nextToken')
             if not next_token:
                 break
         return all_log_events
 
-    def _extract_account_number(self, ses_domain_arn):
-        account_number = ses_domain_arn.split(":")
-        return account_number
-
-    def warn_if_dev_is_opted_out(self, provider_response, notification_id):
-        if (
-            "is opted out" in provider_response.lower()
-            or "has blocked sms" in provider_response.lower()
-        ):
-            if os.getenv("NOTIFY_ENVIRONMENT") in ["development", "test"]:
-                ansi_red = "\033[31m"
-                ansi_reset = "\033[0m"
-                logline = (
-                    ansi_red
-                    + f"The phone number for notification_id {notification_id} is OPTED OUT. You need to opt back in"
-                    + ansi_reset
-                )
-                current_app.logger.warning(logline)
-                return logline
-        return None
-
     def check_sms(self, message_id, notification_id, created_at):
-        region = cloud_config.sns_region
-        # TODO this clumsy approach to getting the account number will be fixed as part of notify-api #258
-        account_number = self._extract_account_number(cloud_config.ses_domain_arn)
 
-        time_now = utc_now()
-        log_group_name = f"sns/{region}/{account_number[4]}/DirectPublishToPhoneNumber"
+        # TODO this clumsy approach to getting the account number will be fixed as part of notify-api #258
+        account_number = cloud_config.ses_domain_arn
+        account_number = account_number.replace('arn:aws:ses:us-west-2:', '')
+        account_number = account_number.split(":")
+        account_number = account_number[0]
+
+        log_group_name = f'sns/us-west-2/{account_number}/DirectPublishToPhoneNumber'
         filter_pattern = '{$.notification.messageId="XXXXX"}'
         filter_pattern = filter_pattern.replace("XXXXX", message_id)
         all_log_events = self._get_log(filter_pattern, log_group_name, created_at)
+
         if all_log_events and len(all_log_events) > 0:
             event = all_log_events[0]
-            message = json.loads(event["message"])
-            self.warn_if_dev_is_opted_out(
-                message["delivery"]["providerResponse"], notification_id
-            )
-            # Here we map the answer from aws to the message_id.
-            # Previously, in send_to_providers, we mapped the job_id and row number
-            # to the message id.  And on the admin side we mapped the csv filename
-            # to the job_id.  So by tracing through all the logs we can go:
-            # filename->job_id->message_id->what really happened
-            current_app.logger.info(
-                hilite(f"DELIVERED: {message} for message_id {message_id}")
-            )
-            return (
-                "success",
-                message["delivery"]["providerResponse"],
-                message["delivery"].get("phoneCarrier", "Unknown Carrier"),
-            )
+            message = json.loads(event['message'])
+            return "success", message['delivery']['providerResponse']
 
-        log_group_name = (
-            f"sns/{region}/{account_number[4]}/DirectPublishToPhoneNumber/Failure"
-        )
+        log_group_name = f'sns/us-west-2/{account_number}/DirectPublishToPhoneNumber/Failure'
         all_failed_events = self._get_log(filter_pattern, log_group_name, created_at)
         if all_failed_events and len(all_failed_events) > 0:
             event = all_failed_events[0]
-            message = json.loads(event["message"])
-            self.warn_if_dev_is_opted_out(
-                message["delivery"]["providerResponse"], notification_id
-            )
+            message = json.loads(event['message'])
+            return "failure", message['delivery']['providerResponse']
 
-            current_app.logger.info(
-                hilite(f"FAILED: {message} for message_id {message_id}")
-            )
-            return (
-                "failure",
-                message["delivery"]["providerResponse"],
-                message["delivery"].get("phoneCarrier", "Unknown Carrier"),
-            )
-
-        if time_now > (created_at + timedelta(hours=3)):
-            # see app/models.py Notification. This message corresponds to "permanent-failure",
-            # but we are copy/pasting here to avoid circular imports.
-            return "failure", "Unable to find carrier response."
-        raise NotificationTechnicalFailureException(
-            f"No event found for message_id {message_id} notification_id {notification_id}"
-        )
+        raise Exception(f'No event found for message_id {message_id} notification_id {notification_id}')

app/clients/document_download.py

@@ -9,15 +9,16 @@ class DocumentDownloadError(Exception):
 
     @classmethod
     def from_exception(cls, e):
-        message = e.response.json()["error"]
+        message = e.response.json()['error']
         status_code = e.response.status_code
         return cls(message, status_code)
 
 
 class DocumentDownloadClient:
+
     def init_app(self, app):
-        self.api_host = app.config["DOCUMENT_DOWNLOAD_API_HOST"]
-        self.auth_token = app.config["DOCUMENT_DOWNLOAD_API_KEY"]
+        self.api_host = app.config['DOCUMENT_DOWNLOAD_API_HOST']
+        self.auth_token = app.config['DOCUMENT_DOWNLOAD_API_KEY']
 
     def get_upload_url(self, service_id):
         return "{}/services/{}/documents".format(self.api_host, service_id)
@@ -27,12 +28,12 @@ class DocumentDownloadClient:
             response = requests.post(
                 self.get_upload_url(service_id),
                 headers={
-                    "Authorization": "Bearer {}".format(self.auth_token),
+                    'Authorization': "Bearer {}".format(self.auth_token),
                 },
                 json={
-                    "document": file_contents,
-                    "is_csv": is_csv or False,
-                },
+                    'document': file_contents,
+                    'is_csv': is_csv or False,
+                }
             )
 
             response.raise_for_status()
@@ -41,16 +42,14 @@ class DocumentDownloadClient:
             # we don't want to tell users about that, so anything that isn't a 400 (virus scan failed or file type
             # unrecognised) should be raised as a 500 internal server error here.
             if e.response is None:
-                raise Exception(f"Unhandled document download error: {repr(e)}")
+                raise Exception(f'Unhandled document download error: {repr(e)}')
             elif e.response.status_code == 400:
                 error = DocumentDownloadError.from_exception(e)
                 current_app.logger.info(
-                    "Document download request failed with error: {}".format(
-                        error.message
-                    )
+                    'Document download request failed with error: {}'.format(error.message)
                 )
                 raise error
             else:
-                raise Exception(f"Unhandled document download error: {e.response.text}")
+                raise Exception(f'Unhandled document download error: {e.response.text}')
 
-        return response.json()["document"]["url"]
+        return response.json()['document']['url']

app/clients/email/__init__.py

@@ -1,38 +1,33 @@
-from abc import abstractmethod, abstractproperty
-
 from app.clients import Client, ClientException
 
 
 class EmailClientException(ClientException):
-    """
+    '''
     Base Exception for EmailClients
-    """
-
+    '''
     pass
 
 
 class EmailClientNonRetryableException(ClientException):
-    """
+    '''
     Represents an error returned from the email client API with a 4xx response code
     that should not be retried and should instead be marked as technical failure.
     An example of this would be an email address that makes it through our
     validation rules but is rejected by SES. There is no point in retrying this type as
     it will always fail however many calls to SES. Whereas a throttling error would not
     use this exception as it may succeed if we retry
-    """
-
+    '''
     pass
 
 
 class EmailClient(Client):
-    """
+    '''
     Base Email client for sending emails.
-    """
+    '''
 
-    @abstractmethod
     def send_email(self, *args, **kwargs):
-        raise NotImplementedError("TODO Need to implement.")
+        raise NotImplementedError('TODO Need to implement.')
 
-    @abstractproperty
+    @property
     def name(self):
-        raise NotImplementedError("TODO Need to implement.")
+        raise NotImplementedError('TODO Need to implement.')

app/clients/email/aws_ses.py

@@ -4,40 +4,39 @@ import botocore
 from boto3 import client
 from flask import current_app
 
-from app.clients import AWS_CLIENT_CONFIG
+from app.clients import STATISTICS_DELIVERED, STATISTICS_FAILURE
 from app.clients.email import (
     EmailClient,
     EmailClientException,
     EmailClientNonRetryableException,
 )
 from app.cloudfoundry_config import cloud_config
-from app.enums import NotificationStatus, StatisticsType
 
 ses_response_map = {
-    "Permanent": {
-        "message": "Hard bounced",
+    'Permanent': {
+        "message": 'Hard bounced',
         "success": False,
-        "notification_status": NotificationStatus.PERMANENT_FAILURE,
-        "notification_statistics_status": StatisticsType.FAILURE,
+        "notification_status": 'permanent-failure',
+        "notification_statistics_status": STATISTICS_FAILURE
     },
-    "Temporary": {
-        "message": "Soft bounced",
+    'Temporary': {
+        "message": 'Soft bounced',
         "success": False,
-        "notification_status": NotificationStatus.TEMPORARY_FAILURE,
-        "notification_statistics_status": StatisticsType.FAILURE,
+        "notification_status": 'temporary-failure',
+        "notification_statistics_status": STATISTICS_FAILURE
     },
-    "Delivery": {
-        "message": "Delivered",
+    'Delivery': {
+        "message": 'Delivered',
         "success": True,
-        "notification_status": NotificationStatus.DELIVERED,
-        "notification_statistics_status": StatisticsType.DELIVERED,
+        "notification_status": 'delivered',
+        "notification_statistics_status": STATISTICS_DELIVERED
     },
-    "Complaint": {
-        "message": "Complaint",
+    'Complaint': {
+        "message": 'Complaint',
         "success": True,
-        "notification_status": NotificationStatus.DELIVERED,
-        "notification_statistics_status": StatisticsType.DELIVERED,
-    },
+        "notification_status": 'delivered',
+        "notification_statistics_status": STATISTICS_DELIVERED
+    }
 }
 
 
@@ -54,85 +53,82 @@ class AwsSesClientThrottlingSendRateException(AwsSesClientException):
 
 
 class AwsSesClient(EmailClient):
-    """
+    '''
     Amazon SES email client.
-    """
+    '''
 
     def init_app(self, *args, **kwargs):
         self._client = client(
-            "ses",
+            'ses',
             region_name=cloud_config.ses_region,
             aws_access_key_id=cloud_config.ses_access_key,
-            aws_secret_access_key=cloud_config.ses_secret_key,
-            config=AWS_CLIENT_CONFIG,
+            aws_secret_access_key=cloud_config.ses_secret_key
         )
         super(AwsSesClient, self).__init__(*args, **kwargs)
 
     @property
     def name(self):
-        return "ses"
+        return 'ses'
 
-    def send_email(
-        self, source, to_addresses, subject, body, html_body="", reply_to_address=None
-    ):
+    def send_email(self,
+                   source,
+                   to_addresses,
+                   subject,
+                   body,
+                   html_body='',
+                   reply_to_address=None):
         try:
             if isinstance(to_addresses, str):
                 to_addresses = [to_addresses]
 
             reply_to_addresses = [reply_to_address] if reply_to_address else []
 
-            body = {"Text": {"Data": body}}
+            body = {
+                'Text': {'Data': body}
+            }
 
             if html_body:
-                body.update({"Html": {"Data": html_body}})
+                body.update({
+                    'Html': {'Data': html_body}
+                })
 
             start_time = monotonic()
             response = self._client.send_email(
                 Source=source,
                 Destination={
-                    "ToAddresses": [
-                        punycode_encode_email(addr) for addr in to_addresses
-                    ],
-                    "CcAddresses": [],
-                    "BccAddresses": [],
+                    'ToAddresses': [punycode_encode_email(addr) for addr in to_addresses],
+                    'CcAddresses': [],
+                    'BccAddresses': []
                 },
                 Message={
-                    "Subject": {
-                        "Data": subject,
+                    'Subject': {
+                        'Data': subject,
                     },
-                    "Body": body,
+                    'Body': body
                 },
-                ReplyToAddresses=[
-                    punycode_encode_email(addr) for addr in reply_to_addresses
-                ],
+                ReplyToAddresses=[punycode_encode_email(addr) for addr in reply_to_addresses]
             )
         except botocore.exceptions.ClientError as e:
-            _do_fancy_exception_handling(e)
 
+            # http://docs.aws.amazon.com/ses/latest/DeveloperGuide/api-error-codes.html
+            if e.response['Error']['Code'] == 'InvalidParameterValue':
+                raise EmailClientNonRetryableException(e.response['Error']['Message'])
+            elif (
+                e.response['Error']['Code'] == 'Throttling'
+                and e.response['Error']['Message'] == 'Maximum sending rate exceeded.'
+            ):
+                raise AwsSesClientThrottlingSendRateException(str(e))
+            else:
+                raise AwsSesClientException(str(e))
         except Exception as e:
             raise AwsSesClientException(str(e))
         else:
             elapsed_time = monotonic() - start_time
-            current_app.logger.info(
-                "AWS SES request finished in {}".format(elapsed_time)
-            )
-            return response["MessageId"]
+            current_app.logger.info("AWS SES request finished in {}".format(elapsed_time))
+            return response['MessageId']
 
 
 def punycode_encode_email(email_address):
     # only the hostname should ever be punycode encoded.
-    local, hostname = email_address.split("@")
-    return "{}@{}".format(local, hostname.encode("idna").decode("utf-8"))
-
-
-def _do_fancy_exception_handling(e):
-    # http://docs.aws.amazon.com/ses/latest/DeveloperGuide/api-error-codes.html
-    if e.response["Error"]["Code"] == "InvalidParameterValue":
-        raise EmailClientNonRetryableException(e.response["Error"]["Message"])
-    elif (
-        e.response["Error"]["Code"] == "Throttling"
-        and e.response["Error"]["Message"] == "Maximum sending rate exceeded."
-    ):
-        raise AwsSesClientThrottlingSendRateException(str(e))
-    else:
-        raise AwsSesClientException(str(e))
+    local, hostname = email_address.split('@')
+    return '{}@{}'.format(local, hostname.encode('idna').decode('utf-8'))

app/clients/email/aws_ses_stub.py

@@ -17,14 +17,23 @@ class AwsSesStubClient(EmailClient):
 
     @property
     def name(self):
-        return "ses"
+        return 'ses'
 
-    def send_email(
-        self, source, to_addresses, subject, body, html_body="", reply_to_address=None
-    ):
+    def send_email(self,
+                   source,
+                   to_addresses,
+                   subject,
+                   body,
+                   html_body='',
+                   reply_to_address=None):
         try:
             start_time = monotonic()
-            response = request("POST", self.url, data={"id": "dummy-data"}, timeout=60)
+            response = request(
+                "POST",
+                self.url,
+                data={"id": "dummy-data"},
+                timeout=60
+            )
             response.raise_for_status()
             response_json = json.loads(response.text)
 
@@ -32,7 +41,5 @@ class AwsSesStubClient(EmailClient):
             raise AwsSesStubClientException(str(e))
         else:
             elapsed_time = monotonic() - start_time
-            current_app.logger.info(
-                "AWS SES stub request finished in {}".format(elapsed_time)
-            )
-            return response_json["MessageId"]
+            current_app.logger.info("AWS SES stub request finished in {}".format(elapsed_time))
+            return response_json['MessageId']

app/clients/performance_platform/performance_platform_client.py

@@ -6,49 +6,45 @@ from flask import current_app
 
 
 class PerformancePlatformClient:
+
     @property
     def active(self):
         return self._active
 
-    def init_app(self, app, *args, **kwargs):
-        self._active = app.config.get("PERFORMANCE_PLATFORM_ENABLED")
+    def init_app(self, app):
+        self._active = app.config.get('PERFORMANCE_PLATFORM_ENABLED')
         if self.active:
-            self.performance_platform_url = app.config.get("PERFORMANCE_PLATFORM_URL")
-            self.performance_platform_endpoints = app.config.get(
-                "PERFORMANCE_PLATFORM_ENDPOINTS"
-            )
+            self.performance_platform_url = app.config.get('PERFORMANCE_PLATFORM_URL')
+            self.performance_platform_endpoints = app.config.get('PERFORMANCE_PLATFORM_ENDPOINTS')
 
     def send_stats_to_performance_platform(self, payload):
         if self.active:
-            bearer_token = self.performance_platform_endpoints[payload["dataType"]]
+            bearer_token = self.performance_platform_endpoints[payload['dataType']]
             headers = {
-                "Content-Type": "application/json",
-                "Authorization": "Bearer {}".format(bearer_token),
+                'Content-Type': "application/json",
+                'Authorization': 'Bearer {}'.format(bearer_token)
             }
             resp = requests.post(
-                self.performance_platform_url + payload["dataType"],
+                self.performance_platform_url + payload['dataType'],
                 json=payload,
-                headers=headers,
+                headers=headers
             )
 
             if resp.status_code == 200:
                 current_app.logger.info(
-                    "Updated performance platform successfully with payload {}".format(
-                        json.dumps(payload)
-                    )
+                    "Updated performance platform successfully with payload {}".format(json.dumps(payload))
                 )
             else:
                 current_app.logger.error(
                     "Performance platform update request failed for payload with response details: {} '{}'".format(
-                        json.dumps(payload), resp.status_code
+                        json.dumps(payload),
+                        resp.status_code
                     )
                 )
                 resp.raise_for_status()
 
     @staticmethod
-    def format_payload(
-        *, dataset, start_time, group_name, group_value, count, period="day"
-    ):
+    def format_payload(*, dataset, start_time, group_name, group_value, count, period='day'):
         """
         :param dataset - the name of the overall graph, as referred to in the endpoint.
         :param start_time - UTC midnight of the day we're sending stats for
@@ -58,16 +54,14 @@ class PerformancePlatformClient:
         :param period - the period that this data covers - "day", "week", "month", "quarter".
         """
         payload = {
-            "_timestamp": start_time,
-            "service": "govuk-notify",
-            "dataType": dataset,
-            "period": period,
-            "count": count,
+            '_timestamp': start_time,
+            'service': 'govuk-notify',
+            'dataType': dataset,
+            'period': period,
+            'count': count,
             group_name: group_value,
         }
-        payload["_id"] = PerformancePlatformClient.generate_payload_id(
-            payload, group_name
-        )
+        payload['_id'] = PerformancePlatformClient.generate_payload_id(payload, group_name)
         return payload
 
     @staticmethod
@@ -75,12 +69,12 @@ class PerformancePlatformClient:
         """
         group_name is the name of the group - eg "channel" or "status"
         """
-        payload_string = "{}{}{}{}{}".format(
-            payload["_timestamp"],
-            payload["service"],
+        payload_string = '{}{}{}{}{}'.format(
+            payload['_timestamp'],
+            payload['service'],
             payload[group_name],
-            payload["dataType"],
-            payload["period"],
+            payload['dataType'],
+            payload['period']
         )
-        _id = base64.b64encode(payload_string.encode("utf-8"))
-        return _id.decode("utf-8")
+        _id = base64.b64encode(payload_string.encode('utf-8'))
+        return _id.decode('utf-8')

app/clients/sms/__init__.py

@@ -1,6 +1,3 @@
-from abc import abstractmethod, abstractproperty
-from typing import final
-
 from app.clients import Client, ClientException
 
 
@@ -13,7 +10,7 @@ class SmsClientResponseException(ClientException):
         self.message = message
 
     def __str__(self):
-        return f"Message {self.message}"
+        return "Message {}".format(self.message)
 
 
 class SmsClient(Client):
@@ -21,14 +18,11 @@ class SmsClient(Client):
     Base Sms client for sending smss.
     """
 
-    @abstractmethod
+    def init_app(self, *args, **kwargs):
+        raise NotImplementedError("TODO Need to implement.")
+
     def send_sms(self, *args, **kwargs):
         raise NotImplementedError("TODO Need to implement.")
 
-    @abstractproperty
-    def name(self):
-        raise NotImplementedError("TODO Need to implement.")
-
-    @final
     def get_name(self):
-        return self.name
+        raise NotImplementedError("TODO Need to implement.")

app/clients/sms/aws_sns.py

@@ -1,4 +1,3 @@
-import os
 import re
 from time import monotonic
 
@@ -6,7 +5,6 @@ import botocore
 import phonenumbers
 from boto3 import client
 
-from app.clients import AWS_CLIENT_CONFIG
 from app.clients.sms import SmsClient
 from app.cloudfoundry_config import cloud_config
 
@@ -17,31 +15,22 @@ class AwsSnsClient(SmsClient):
     """
 
     def init_app(self, current_app, *args, **kwargs):
-        if os.getenv("LOCALSTACK_ENDPOINT_URL"):
-            self._client = client(
-                "sns",
-                region_name=cloud_config.sns_region,
-                aws_access_key_id=cloud_config.sns_access_key,
-                aws_secret_access_key=cloud_config.sns_secret_key,
-                config=AWS_CLIENT_CONFIG,
-                endpoint_url=os.getenv("LOCALSTACK_ENDPOINT_URL"),
-            )
-        else:
-            self._client = client(
-                "sns",
-                region_name=cloud_config.sns_region,
-                aws_access_key_id=cloud_config.sns_access_key,
-                aws_secret_access_key=cloud_config.sns_secret_key,
-                config=AWS_CLIENT_CONFIG,
-            )
-
+        self._client = client(
+            "sns",
+            region_name=cloud_config.sns_region,
+            aws_access_key_id=cloud_config.sns_access_key,
+            aws_secret_access_key=cloud_config.sns_secret_key
+        )
         super(SmsClient, self).__init__(*args, **kwargs)
         self.current_app = current_app
         self._valid_sender_regex = re.compile(r"^\+?\d{5,14}$")
 
     @property
     def name(self):
-        return "sns"
+        return 'sns'
+
+    def get_name(self):
+        return self.name
 
     def _valid_sender_number(self, sender):
         return sender and re.match(self._valid_sender_regex, sender)
@@ -51,9 +40,7 @@ class AwsSnsClient(SmsClient):
 
         for match in phonenumbers.PhoneNumberMatcher(to, "US"):
             matched = True
-            to = phonenumbers.format_number(
-                match.number, phonenumbers.PhoneNumberFormat.E164
-            )
+            to = phonenumbers.format_number(match.number, phonenumbers.PhoneNumberFormat.E164)
 
             # See documentation
             # https://docs.aws.amazon.com/sns/latest/dg/sms_publish-to-phone.html#sms_publish_sdk
@@ -77,20 +64,14 @@ class AwsSnsClient(SmsClient):
 
             try:
                 start_time = monotonic()
-                response = self._client.publish(
-                    PhoneNumber=to, Message=content, MessageAttributes=attributes
-                )
+                response = self._client.publish(PhoneNumber=to, Message=content, MessageAttributes=attributes)
             except botocore.exceptions.ClientError as e:
-                self.current_app.logger.error(e)
                 raise str(e)
             except Exception as e:
-                self.current_app.logger(e)
                 raise str(e)
             finally:
                 elapsed_time = monotonic() - start_time
-                self.current_app.logger.info(
-                    "AWS SNS request finished in {}".format(elapsed_time)
-                )
+                self.current_app.logger.info("AWS SNS request finished in {}".format(elapsed_time))
             return response["MessageId"]
 
         if not matched:

app/cloudfoundry_config.py

@@ -4,28 +4,29 @@ from os import getenv
 
 class CloudfoundryConfig:
     def __init__(self):
-        self.parsed_services = json.loads(getenv("VCAP_SERVICES") or "{}")
-        buckets = self.parsed_services.get("s3") or []
-        self.s3_buckets = {bucket["name"]: bucket["credentials"] for bucket in buckets}
+        self.parsed_services = json.loads(getenv('VCAP_SERVICES') or '{}')
+        buckets = self.parsed_services.get('s3') or []
+        self.s3_buckets = {bucket['name']: bucket['credentials'] for bucket in buckets}
         self._empty_bucket_credentials = {
-            "bucket": "",
-            "access_key_id": "",
-            "secret_access_key": "",
-            "region": "",
+            'bucket': '',
+            'access_key_id': '',
+            'secret_access_key': '',
+            'region': ''
         }
 
     @property
     def database_url(self):
-        return getenv("DATABASE_URL", "").replace("postgres://", "postgresql://")
+        return getenv('DATABASE_URL', '').replace('postgres://', 'postgresql://')
 
     @property
     def redis_url(self):
         try:
-            return self.parsed_services["aws-elasticache-redis"][0]["credentials"][
-                "uri"
-            ].replace("redis://", "rediss://")
+            return self.parsed_services['aws-elasticache-redis'][0]['credentials']['uri'].replace(
+                'redis://',
+                'rediss://'
+            )
         except KeyError:
-            return getenv("REDIS_URL")
+            return getenv('REDIS_URL')
 
     def s3_credentials(self, service_name):
         return self.s3_buckets.get(service_name) or self._empty_bucket_credentials
@@ -33,78 +34,78 @@ class CloudfoundryConfig:
     @property
     def ses_email_domain(self):
         try:
-            domain_arn = self._ses_credentials("domain_arn")
+            domain_arn = self._ses_credentials('domain_arn')
         except KeyError:
-            domain_arn = getenv("SES_DOMAIN_ARN", "dev.notify.gov")
-        return domain_arn.split("/")[-1]
+            domain_arn = getenv('SES_DOMAIN_ARN', 'dev.notify.gov')
+        return domain_arn.split('/')[-1]
 
     # TODO remove this after notifications-api #258
     @property
     def ses_domain_arn(self):
         try:
-            domain_arn = self._ses_credentials("domain_arn")
+            domain_arn = self._ses_credentials('domain_arn')
         except KeyError:
-            domain_arn = getenv("SES_DOMAIN_ARN", "dev.notify.gov")
+            domain_arn = getenv('SES_DOMAIN_ARN', 'dev.notify.gov')
         return domain_arn
 
     @property
     def ses_region(self):
         try:
-            return self._ses_credentials("region")
+            return self._ses_credentials('region')
         except KeyError:
-            return getenv("SES_AWS_REGION", "us-west-1")
+            return getenv('SES_AWS_REGION', 'us-west-1')
 
     @property
     def ses_access_key(self):
         try:
-            return self._ses_credentials("smtp_user")
+            return self._ses_credentials('smtp_user')
         except KeyError:
-            return getenv("SES_AWS_ACCESS_KEY_ID")
+            return getenv('SES_AWS_ACCESS_KEY_ID')
 
     @property
     def ses_secret_key(self):
         try:
-            return self._ses_credentials("secret_access_key")
+            return self._ses_credentials('secret_access_key')
         except KeyError:
-            return getenv("SES_AWS_SECRET_ACCESS_KEY")
+            return getenv('SES_AWS_SECRET_ACCESS_KEY')
 
     @property
     def sns_access_key(self):
         try:
-            return self._sns_credentials("aws_access_key_id")
+            return self._sns_credentials('aws_access_key_id')
         except KeyError:
-            return getenv("SNS_AWS_ACCESS_KEY_ID")
+            return getenv('SNS_AWS_ACCESS_KEY_ID')
 
     @property
     def sns_secret_key(self):
         try:
-            return self._sns_credentials("aws_secret_access_key")
+            return self._sns_credentials('aws_secret_access_key')
         except KeyError:
-            return getenv("SNS_AWS_SECRET_ACCESS_KEY")
+            return getenv('SNS_AWS_SECRET_ACCESS_KEY')
 
     @property
     def sns_region(self):
         try:
-            return self._sns_credentials("region")
+            return self._sns_credentials('region')
         except KeyError:
-            return getenv("SNS_AWS_REGION", "us-west-1")
+            return getenv('SNS_AWS_REGION', 'us-west-1')
 
     @property
     def sns_topic_arns(self):
         try:
             return [
-                self._ses_credentials("bounce_topic_arn"),
-                self._ses_credentials("complaint_topic_arn"),
-                self._ses_credentials("delivery_topic_arn"),
+                self._ses_credentials('bounce_topic_arn'),
+                self._ses_credentials('complaint_topic_arn'),
+                self._ses_credentials('delivery_topic_arn')
             ]
         except KeyError:
             return []
 
     def _ses_credentials(self, key):
-        return self.parsed_services["datagov-smtp"][0]["credentials"][key]
+        return self.parsed_services['datagov-smtp'][0]['credentials'][key]
 
     def _sns_credentials(self, key):
-        return self.parsed_services["ttsnotify-sms"][0]["credentials"][key]
+        return self.parsed_services['ttsnotify-sms'][0]['credentials'][key]
 
 
 cloud_config = CloudfoundryConfig()

app/complaint/complaint_rest.py

@@ -3,46 +3,44 @@ from datetime import datetime
 from flask import Blueprint, jsonify, request
 
 from app.complaint.complaint_schema import complaint_count_request
-from app.dao.complaint_dao import fetch_count_of_complaints, fetch_paginated_complaints
+from app.dao.complaint_dao import (
+    fetch_count_of_complaints,
+    fetch_paginated_complaints,
+)
 from app.errors import register_errors
 from app.schema_validation import validate
-from app.utils import pagination_links, utc_now
+from app.utils import pagination_links
 
-complaint_blueprint = Blueprint("complaint", __name__, url_prefix="/complaint")
+complaint_blueprint = Blueprint('complaint', __name__, url_prefix='/complaint')
 
 register_errors(complaint_blueprint)
 
 
-@complaint_blueprint.route("", methods=["GET"])
+@complaint_blueprint.route('', methods=['GET'])
 def get_all_complaints():
-    page = int(request.args.get("page", 1))
+    page = int(request.args.get('page', 1))
     pagination = fetch_paginated_complaints(page=page)
 
-    return (
-        jsonify(
-            complaints=[x.serialize() for x in pagination.items],
-            links=pagination_links(
-                pagination, ".get_all_complaints", **request.args.to_dict()
-            ),
-        ),
-        200,
-    )
+    return jsonify(
+        complaints=[x.serialize() for x in pagination.items],
+        links=pagination_links(
+            pagination,
+            '.get_all_complaints',
+            **request.args.to_dict()
+        )
+    ), 200
 
 
-@complaint_blueprint.route("/count-by-date-range", methods=["GET"])
+@complaint_blueprint.route('/count-by-date-range', methods=['GET'])
 def get_complaint_count():
     if request.args:
         validate(request.args, complaint_count_request)
 
     # If start and end date are not set, we are expecting today's stats.
-    today = str(utc_now().date())
+    today = str(datetime.utcnow().date())
 
-    start_date = datetime.strptime(
-        request.args.get("start_date", today), "%Y-%m-%d"
-    ).date()
-    end_date = datetime.strptime(request.args.get("end_date", today), "%Y-%m-%d").date()
-    count_of_complaints = fetch_count_of_complaints(
-        start_date=start_date, end_date=end_date
-    )
+    start_date = datetime.strptime(request.args.get('start_date', today), '%Y-%m-%d').date()
+    end_date = datetime.strptime(request.args.get('end_date', today), '%Y-%m-%d').date()
+    count_of_complaints = fetch_count_of_complaints(start_date=start_date, end_date=end_date)
 
     return jsonify(count_of_complaints), 200

app/complaint/complaint_schema.py

@@ -1,3 +1,4 @@
+
 complaint_count_request = {
     "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "complaint count request schema",
@@ -6,5 +7,5 @@ complaint_count_request = {
     "properties": {
         "start_date": {"type": ["string", "null"], "format": "date"},
         "end_date": {"type": ["string", "null"], "format": "date"},
-    },
+    }
 }

app/config.py

@@ -2,30 +2,31 @@ import json
 from datetime import timedelta
 from os import getenv, path
 
+import notifications_utils
 from celery.schedules import crontab
 from kombu import Exchange, Queue
 
-import notifications_utils
 from app.cloudfoundry_config import cloud_config
 
 
 class QueueNames(object):
-    PERIODIC = "periodic-tasks"
-    PRIORITY = "priority-tasks"
-    DATABASE = "database-tasks"
-    SEND_SMS = "send-sms-tasks"
-    CHECK_SMS = "check-sms_tasks"
-    SEND_EMAIL = "send-email-tasks"
-    REPORTING = "reporting-tasks"
-    JOBS = "job-tasks"
-    RETRY = "retry-tasks"
-    NOTIFY = "notify-internal-tasks"
-    CALLBACKS = "service-callbacks"
-    CALLBACKS_RETRY = "service-callbacks-retry"
-    SMS_CALLBACKS = "sms-callbacks"
-    ANTIVIRUS = "antivirus-tasks"
-    SAVE_API_EMAIL = "save-api-email-tasks"
-    SAVE_API_SMS = "save-api-sms-tasks"
+    PERIODIC = 'periodic-tasks'
+    PRIORITY = 'priority-tasks'
+    DATABASE = 'database-tasks'
+    SEND_SMS = 'send-sms-tasks'
+    CHECK_SMS = 'check-sms_tasks'
+    SEND_EMAIL = 'send-email-tasks'
+    RESEARCH_MODE = 'research-mode-tasks'
+    REPORTING = 'reporting-tasks'
+    JOBS = 'job-tasks'
+    RETRY = 'retry-tasks'
+    NOTIFY = 'notify-internal-tasks'
+    CALLBACKS = 'service-callbacks'
+    CALLBACKS_RETRY = 'service-callbacks-retry'
+    SMS_CALLBACKS = 'sms-callbacks'
+    ANTIVIRUS = 'antivirus-tasks'
+    SAVE_API_EMAIL = 'save-api-email-tasks'
+    SAVE_API_SMS = 'save-api-sms-tasks'
 
     @staticmethod
     def all_queues():
@@ -36,6 +37,7 @@ class QueueNames(object):
             QueueNames.SEND_SMS,
             QueueNames.CHECK_SMS,
             QueueNames.SEND_EMAIL,
+            QueueNames.RESEARCH_MODE,
             QueueNames.REPORTING,
             QueueNames.JOBS,
             QueueNames.RETRY,
@@ -49,47 +51,47 @@ class QueueNames(object):
 
 
 class TaskNames(object):
-    PROCESS_INCOMPLETE_JOBS = "process-incomplete-jobs"
-    SCAN_FILE = "scan-file"
+    PROCESS_INCOMPLETE_JOBS = 'process-incomplete-jobs'
+    SCAN_FILE = 'scan-file'
 
 
 class Config(object):
-    NOTIFY_APP_NAME = "api"
-    NOTIFY_ENVIRONMENT = getenv("NOTIFY_ENVIRONMENT", "development")
+    NOTIFY_APP_NAME = 'api'
+    NOTIFY_ENVIRONMENT = getenv('NOTIFY_ENVIRONMENT', 'development')
     # URL of admin app
-    ADMIN_BASE_URL = getenv("ADMIN_BASE_URL", "http://localhost:6012")
+    ADMIN_BASE_URL = getenv('ADMIN_BASE_URL', 'http://localhost:6012')
     # URL of api app (on AWS this is the internal api endpoint)
-    API_HOST_NAME = getenv("API_HOST_NAME", "http://localhost:6011")
+    API_HOST_NAME = getenv('API_HOST_NAME', 'http://localhost:6011')
 
     # Credentials
     # secrets that internal apps, such as the admin app or document download, must use to authenticate with the API
     # ADMIN_CLIENT_ID is called ADMIN_CLIENT_USER_NAME in api repo, they should match
-    ADMIN_CLIENT_ID = getenv("ADMIN_CLIENT_ID", "notify-admin")
+    ADMIN_CLIENT_ID = getenv('ADMIN_CLIENT_ID', 'notify-admin')
     INTERNAL_CLIENT_API_KEYS = json.loads(
         getenv(
-            "INTERNAL_CLIENT_API_KEYS",
-            ('{"%s":["%s"]}' % (ADMIN_CLIENT_ID, getenv("ADMIN_CLIENT_SECRET"))),
-        )
+            'INTERNAL_CLIENT_API_KEYS',
+            ('{"%s":["%s"]}' % (ADMIN_CLIENT_ID, getenv('ADMIN_CLIENT_SECRET')))
+            )
     )
     ALLOW_EXPIRED_API_TOKEN = False
     # encyption secret/salt
-    SECRET_KEY = getenv("SECRET_KEY")
-    DANGEROUS_SALT = getenv("DANGEROUS_SALT")
-    ROUTE_SECRET_KEY_1 = getenv("ROUTE_SECRET_KEY_1", "dev-route-secret-key-1")
-    ROUTE_SECRET_KEY_2 = getenv("ROUTE_SECRET_KEY_2", "dev-route-secret-key-2")
+    SECRET_KEY = getenv('SECRET_KEY')
+    DANGEROUS_SALT = getenv('DANGEROUS_SALT')
+    ROUTE_SECRET_KEY_1 = getenv('ROUTE_SECRET_KEY_1', 'dev-route-secret-key-1')
+    ROUTE_SECRET_KEY_2 = getenv('ROUTE_SECRET_KEY_2', 'dev-route-secret-key-2')
 
     # DB settings
     SQLALCHEMY_DATABASE_URI = cloud_config.database_url
     SQLALCHEMY_RECORD_QUERIES = False
     SQLALCHEMY_TRACK_MODIFICATIONS = False
-    SQLALCHEMY_POOL_SIZE = int(getenv("SQLALCHEMY_POOL_SIZE", 5))
+    SQLALCHEMY_POOL_SIZE = int(getenv('SQLALCHEMY_POOL_SIZE', 5))
     SQLALCHEMY_POOL_TIMEOUT = 30
     SQLALCHEMY_POOL_RECYCLE = 300
     SQLALCHEMY_STATEMENT_TIMEOUT = 1200
     PAGE_SIZE = 50
     API_PAGE_SIZE = 250
     REDIS_URL = cloud_config.redis_url
-    REDIS_ENABLED = getenv("REDIS_ENABLED", "1") == "1"
+    REDIS_ENABLED = getenv('REDIS_ENABLED', '0') == '1'
     EXPIRE_CACHE_TEN_MINUTES = 600
     EXPIRE_CACHE_EIGHT_DAYS = 8 * 24 * 60 * 60
 
@@ -98,214 +100,197 @@ class Config(object):
     # Whether to ignore POSTs from SNS for replies to SMS we sent
     RECEIVE_INBOUND_SMS = False
     NOTIFY_EMAIL_DOMAIN = cloud_config.ses_email_domain
-    SES_STUB_URL = (
-        None  # TODO: set to a URL in env and remove this to use a stubbed SES service
-    )
+    SES_STUB_URL = None  # TODO: set to a URL in env and remove this to use a stubbed SES service
     # AWS SNS topics for delivery receipts
     VALIDATE_SNS_TOPICS = True
     VALID_SNS_TOPICS = cloud_config.sns_topic_arns
 
     # these should always add up to 100%
     SMS_PROVIDER_RESTING_POINTS = {
-        "sns": 100,
+        'sns': 100,
     }
 
     # Zendesk
-    ZENDESK_API_KEY = getenv("ZENDESK_API_KEY")
+    ZENDESK_API_KEY = getenv('ZENDESK_API_KEY')
 
     # Logging
     DEBUG = False
 
     # Monitoring
     CRONITOR_ENABLED = False
-    CRONITOR_KEYS = json.loads(getenv("CRONITOR_KEYS", "{}"))
+    CRONITOR_KEYS = json.loads(getenv('CRONITOR_KEYS', '{}'))
 
     # Antivirus
-    ANTIVIRUS_ENABLED = getenv("ANTIVIRUS_ENABLED", "1") == "1"
+    ANTIVIRUS_ENABLED = getenv('ANTIVIRUS_ENABLED', '1') == '1'
 
     SENDING_NOTIFICATIONS_TIMEOUT_PERIOD = 259200  # 3 days
     INVITATION_EXPIRATION_DAYS = 2
-    TEST_MESSAGE_FILENAME = "Test message"
-    ONE_OFF_MESSAGE_FILENAME = "Report"
+    TEST_MESSAGE_FILENAME = 'Test message'
+    ONE_OFF_MESSAGE_FILENAME = 'Report'
     MAX_VERIFY_CODE_COUNT = 5
     MAX_FAILED_LOGIN_COUNT = 10
     API_RATE_LIMIT_ENABLED = True
 
     # Default data
-    CONFIG_FILES = path.dirname(__file__) + "/config_files/"
+    CONFIG_FILES = path.dirname(__file__) + '/config_files/'
 
-    NOTIFY_SERVICE_ID = "d6aa2c68-a2d9-4437-ab19-3ae8eb202553"
-    NOTIFY_USER_ID = "6af522d0-2915-4e52-83a3-3690455a5fe6"
-    INVITATION_EMAIL_TEMPLATE_ID = "4f46df42-f795-4cc4-83bb-65ca312f49cc"
-    SMS_CODE_TEMPLATE_ID = "36fb0730-6259-4da1-8a80-c8de22ad4246"
-    EMAIL_2FA_TEMPLATE_ID = "299726d2-dba6-42b8-8209-30e1d66ea164"
-    NEW_USER_EMAIL_VERIFICATION_TEMPLATE_ID = "ece42649-22a8-4d06-b87f-d52d5d3f0a27"
-    PASSWORD_RESET_TEMPLATE_ID = (
-        "474e9242-823b-4f99-813d-ed392e7f1201"  # nosec B105 - this is not a password
-    )
-    ALREADY_REGISTERED_EMAIL_TEMPLATE_ID = "0880fbb1-a0c6-46f0-9a8e-36c986381ceb"
-    CHANGE_EMAIL_CONFIRMATION_TEMPLATE_ID = "eb4d9930-87ab-4aef-9bce-786762687884"
-    SERVICE_NOW_LIVE_TEMPLATE_ID = "618185c6-3636-49cd-b7d2-6f6f5eb3bdde"
-    ORGANIZATION_INVITATION_EMAIL_TEMPLATE_ID = "203566f0-d835-47c5-aa06-932439c86573"
-    TEAM_MEMBER_EDIT_EMAIL_TEMPLATE_ID = "c73f1d71-4049-46d5-a647-d013bdeca3f0"
-    TEAM_MEMBER_EDIT_MOBILE_TEMPLATE_ID = "8a31520f-4751-4789-8ea1-fe54496725eb"
-    REPLY_TO_EMAIL_ADDRESS_VERIFICATION_TEMPLATE_ID = (
-        "a42f1d17-9404-46d5-a647-d013bdfca3e1"
-    )
-    MOU_SIGNER_RECEIPT_TEMPLATE_ID = "4fd2e43c-309b-4e50-8fb8-1955852d9d71"
-    MOU_SIGNED_ON_BEHALF_SIGNER_RECEIPT_TEMPLATE_ID = (
-        "c20206d5-bf03-4002-9a90-37d5032d9e84"
-    )
-    MOU_SIGNED_ON_BEHALF_ON_BEHALF_RECEIPT_TEMPLATE_ID = (
-        "522b6657-5ca5-4368-a294-6b527703bd0b"
-    )
-    NOTIFY_INTERNATIONAL_SMS_SENDER = getenv("AWS_US_TOLL_FREE_NUMBER")
-    LETTERS_VOLUME_EMAIL_TEMPLATE_ID = "11fad854-fd38-4a7c-bd17-805fb13dfc12"
-    NHS_EMAIL_BRANDING_ID = "a7dc4e56-660b-4db7-8cff-12c37b12b5ea"
+    NOTIFY_SERVICE_ID = 'd6aa2c68-a2d9-4437-ab19-3ae8eb202553'
+    NOTIFY_USER_ID = '6af522d0-2915-4e52-83a3-3690455a5fe6'
+    INVITATION_EMAIL_TEMPLATE_ID = '4f46df42-f795-4cc4-83bb-65ca312f49cc'
+    SMS_CODE_TEMPLATE_ID = '36fb0730-6259-4da1-8a80-c8de22ad4246'
+    EMAIL_2FA_TEMPLATE_ID = '299726d2-dba6-42b8-8209-30e1d66ea164'
+    NEW_USER_EMAIL_VERIFICATION_TEMPLATE_ID = 'ece42649-22a8-4d06-b87f-d52d5d3f0a27'
+    PASSWORD_RESET_TEMPLATE_ID = '474e9242-823b-4f99-813d-ed392e7f1201'  # nosec B105 - this is not a password
+    ALREADY_REGISTERED_EMAIL_TEMPLATE_ID = '0880fbb1-a0c6-46f0-9a8e-36c986381ceb'
+    CHANGE_EMAIL_CONFIRMATION_TEMPLATE_ID = 'eb4d9930-87ab-4aef-9bce-786762687884'
+    SERVICE_NOW_LIVE_TEMPLATE_ID = '618185c6-3636-49cd-b7d2-6f6f5eb3bdde'
+    ORGANIZATION_INVITATION_EMAIL_TEMPLATE_ID = '203566f0-d835-47c5-aa06-932439c86573'
+    TEAM_MEMBER_EDIT_EMAIL_TEMPLATE_ID = 'c73f1d71-4049-46d5-a647-d013bdeca3f0'
+    TEAM_MEMBER_EDIT_MOBILE_TEMPLATE_ID = '8a31520f-4751-4789-8ea1-fe54496725eb'
+    REPLY_TO_EMAIL_ADDRESS_VERIFICATION_TEMPLATE_ID = 'a42f1d17-9404-46d5-a647-d013bdfca3e1'
+    MOU_SIGNER_RECEIPT_TEMPLATE_ID = '4fd2e43c-309b-4e50-8fb8-1955852d9d71'
+    MOU_SIGNED_ON_BEHALF_SIGNER_RECEIPT_TEMPLATE_ID = 'c20206d5-bf03-4002-9a90-37d5032d9e84'
+    MOU_SIGNED_ON_BEHALF_ON_BEHALF_RECEIPT_TEMPLATE_ID = '522b6657-5ca5-4368-a294-6b527703bd0b'
+    NOTIFY_INTERNATIONAL_SMS_SENDER = getenv('AWS_US_TOLL_FREE_NUMBER')
+    LETTERS_VOLUME_EMAIL_TEMPLATE_ID = '11fad854-fd38-4a7c-bd17-805fb13dfc12'
+    NHS_EMAIL_BRANDING_ID = 'a7dc4e56-660b-4db7-8cff-12c37b12b5ea'
     # we only need real email in Live environment (production)
-    DVLA_EMAIL_ADDRESSES = json.loads(getenv("DVLA_EMAIL_ADDRESSES", "[]"))
+    DVLA_EMAIL_ADDRESSES = json.loads(getenv('DVLA_EMAIL_ADDRESSES', '[]'))
 
     CELERY = {
-        "broker_url": REDIS_URL,
-        "broker_transport_options": {
-            "visibility_timeout": 310,
+        'broker_url': REDIS_URL,
+        'broker_transport_options': {
+            'visibility_timeout': 310,
         },
-        "timezone": getenv("TIMEZONE", "UTC"),
-        "imports": [
-            "app.celery.tasks",
-            "app.celery.scheduled_tasks",
-            "app.celery.reporting_tasks",
-            "app.celery.nightly_tasks",
+        'timezone': getenv("TIMEZONE", 'UTC'),
+        'imports': [
+            'app.celery.tasks',
+            'app.celery.scheduled_tasks',
+            'app.celery.reporting_tasks',
+            'app.celery.nightly_tasks',
         ],
         # this is overriden by the -Q command, but locally, we should read from all queues
-        "task_queues": [
-            Queue(queue, Exchange("default"), routing_key=queue)
-            for queue in QueueNames.all_queues()
+        'task_queues': [
+            Queue(queue, Exchange('default'), routing_key=queue) for queue in QueueNames.all_queues()
         ],
-        "beat_schedule": {
+        'beat_schedule': {
             # app/celery/scheduled_tasks.py
-            "run-scheduled-jobs": {
-                "task": "run-scheduled-jobs",
-                "schedule": crontab(minute="0,15,30,45"),
-                "options": {"queue": QueueNames.PERIODIC},
+            'run-scheduled-jobs': {
+                'task': 'run-scheduled-jobs',
+                'schedule': crontab(minute='0,15,30,45'),
+                'options': {'queue': QueueNames.PERIODIC}
             },
-            "delete-verify-codes": {
-                "task": "delete-verify-codes",
-                "schedule": timedelta(minutes=63),
-                "options": {"queue": QueueNames.PERIODIC},
+            'delete-verify-codes': {
+                'task': 'delete-verify-codes',
+                'schedule': timedelta(minutes=63),
+                'options': {'queue': QueueNames.PERIODIC}
             },
-            "expire-or-delete-invitations": {
-                "task": "expire-or-delete-invitations",
-                "schedule": timedelta(minutes=66),
-                "options": {"queue": QueueNames.PERIODIC},
+            'delete-invitations': {
+                'task': 'delete-invitations',
+                'schedule': timedelta(minutes=66),
+                'options': {'queue': QueueNames.PERIODIC}
             },
-            "check-job-status": {
-                "task": "check-job-status",
-                "schedule": crontab(),
-                "options": {"queue": QueueNames.PERIODIC},
+            'check-job-status': {
+                'task': 'check-job-status',
+                'schedule': crontab(),
+                'options': {'queue': QueueNames.PERIODIC}
             },
-            "check-for-missing-rows-in-completed-jobs": {
-                "task": "check-for-missing-rows-in-completed-jobs",
-                "schedule": crontab(minute="*/10"),
-                "options": {"queue": QueueNames.PERIODIC},
+            'tend-providers-back-to-middle': {
+                'task': 'tend-providers-back-to-middle',
+                'schedule': crontab(minute='*/5'),
+                'options': {'queue': QueueNames.PERIODIC}
             },
-            "replay-created-notifications": {
-                "task": "replay-created-notifications",
-                "schedule": crontab(minute="0, 15, 30, 45"),
-                "options": {"queue": QueueNames.PERIODIC},
+            'check-for-missing-rows-in-completed-jobs': {
+                'task': 'check-for-missing-rows-in-completed-jobs',
+                'schedule': crontab(minute='*/10'),
+                'options': {'queue': QueueNames.PERIODIC}
+            },
+            'replay-created-notifications': {
+                'task': 'replay-created-notifications',
+                'schedule': crontab(minute='0, 15, 30, 45'),
+                'options': {'queue': QueueNames.PERIODIC}
             },
             # app/celery/nightly_tasks.py
-            "timeout-sending-notifications": {
-                "task": "timeout-sending-notifications",
-                "schedule": crontab(hour=4, minute=5),
-                "options": {"queue": QueueNames.PERIODIC},
+            'timeout-sending-notifications': {
+                'task': 'timeout-sending-notifications',
+                'schedule': crontab(hour=4, minute=5),
+                'options': {'queue': QueueNames.PERIODIC}
             },
-            "create-nightly-billing": {
-                "task": "create-nightly-billing",
-                "schedule": crontab(hour=4, minute=15),
-                "options": {"queue": QueueNames.REPORTING},
+            'create-nightly-billing': {
+                'task': 'create-nightly-billing',
+                'schedule': crontab(hour=4, minute=15),
+                'options': {'queue': QueueNames.REPORTING}
             },
-            "create-nightly-notification-status": {
-                "task": "create-nightly-notification-status",
-                "schedule": crontab(
-                    hour=4, minute=30
-                ),  # after 'timeout-sending-notifications'
-                "options": {"queue": QueueNames.REPORTING},
+            'create-nightly-notification-status': {
+                'task': 'create-nightly-notification-status',
+                'schedule': crontab(hour=4, minute=30),  # after 'timeout-sending-notifications'
+                'options': {'queue': QueueNames.REPORTING}
             },
-            "delete-notifications-older-than-retention": {
-                "task": "delete-notifications-older-than-retention",
-                "schedule": crontab(
-                    hour=7, minute=0
-                ),  # after 'create-nightly-notification-status'
-                "options": {"queue": QueueNames.REPORTING},
+            'delete-notifications-older-than-retention': {
+                'task': 'delete-notifications-older-than-retention',
+                'schedule': crontab(hour=7, minute=0),  # after 'create-nightly-notification-status'
+                'options': {'queue': QueueNames.REPORTING}
             },
-            "delete-inbound-sms": {
-                "task": "delete-inbound-sms",
-                "schedule": crontab(hour=5, minute=40),
-                "options": {"queue": QueueNames.PERIODIC},
+            'delete-inbound-sms': {
+                'task': 'delete-inbound-sms',
+                'schedule': crontab(hour=5, minute=40),
+                'options': {'queue': QueueNames.PERIODIC}
             },
-            "save-daily-notification-processing-time": {
-                "task": "save-daily-notification-processing-time",
-                "schedule": crontab(hour=6, minute=0),
-                "options": {"queue": QueueNames.PERIODIC},
+            'save-daily-notification-processing-time': {
+                'task': 'save-daily-notification-processing-time',
+                'schedule': crontab(hour=6, minute=0),
+                'options': {'queue': QueueNames.PERIODIC}
             },
-            "regenerate-job-cache": {
-                "task": "regenerate-job-cache",
-                "schedule": crontab(minute="*/30"),
-                "options": {"queue": QueueNames.PERIODIC},
+            'cleanup-unfinished-jobs': {
+                'task': 'cleanup-unfinished-jobs',
+                'schedule': crontab(hour=4, minute=5),
+                'options': {'queue': QueueNames.PERIODIC}
             },
-            "cleanup-unfinished-jobs": {
-                "task": "cleanup-unfinished-jobs",
-                "schedule": crontab(hour=4, minute=5),
-                "options": {"queue": QueueNames.PERIODIC},
+            'remove_sms_email_jobs': {
+                'task': 'remove_sms_email_jobs',
+                'schedule': crontab(hour=8, minute=0),
+                'options': {'queue': QueueNames.PERIODIC},
             },
-            "remove_sms_email_jobs": {
-                "task": "remove_sms_email_jobs",
-                "schedule": crontab(hour=8, minute=0),
-                "options": {"queue": QueueNames.PERIODIC},
+            'check-for-services-with-high-failure-rates-or-sending-to-tv-numbers': {
+                'task': 'check-for-services-with-high-failure-rates-or-sending-to-tv-numbers',
+                'schedule': crontab(day_of_week='mon-fri', hour=14, minute=30),
+                'options': {'queue': QueueNames.PERIODIC}
             },
-            "check-for-services-with-high-failure-rates-or-sending-to-tv-numbers": {
-                "task": "check-for-services-with-high-failure-rates-or-sending-to-tv-numbers",
-                "schedule": crontab(day_of_week="mon-fri", hour=14, minute=30),
-                "options": {"queue": QueueNames.PERIODIC},
-            },
-        },
+        }
     }
 
     # we can set celeryd_prefetch_multiplier to be 1 for celery apps which handle only long running tasks
-    if getenv("CELERYD_PREFETCH_MULTIPLIER"):
-        CELERY["worker_prefetch_multiplier"] = getenv("CELERYD_PREFETCH_MULTIPLIER")
+    if getenv('CELERYD_PREFETCH_MULTIPLIER'):
+        CELERY['worker_prefetch_multiplier'] = getenv('CELERYD_PREFETCH_MULTIPLIER')
 
-    FROM_NUMBER = "development"
+    FROM_NUMBER = 'development'
 
     SIMULATED_EMAIL_ADDRESSES = (
-        "simulate-delivered@notifications.service.gov.uk",
-        "simulate-delivered-2@notifications.service.gov.uk",
-        "simulate-delivered-3@notifications.service.gov.uk",
+        'simulate-delivered@notifications.service.gov.uk',
+        'simulate-delivered-2@notifications.service.gov.uk',
+        'simulate-delivered-3@notifications.service.gov.uk',
     )
-    # 7755 is success, 7167 is failure
-    SIMULATED_SMS_NUMBERS = ("+14254147755", "+14254147167")
+    SIMULATED_SMS_NUMBERS = ('+12028675000', '+12028675111', '+12028675222')
 
     FREE_SMS_TIER_FRAGMENT_COUNT = 250000
 
-    TOTAL_MESSAGE_LIMIT = 250000
-
     DAILY_MESSAGE_LIMIT = notifications_utils.DAILY_MESSAGE_LIMIT
 
-    HIGH_VOLUME_SERVICE = json.loads(getenv("HIGH_VOLUME_SERVICE", "[]"))
+    HIGH_VOLUME_SERVICE = json.loads(getenv('HIGH_VOLUME_SERVICE', '[]'))
 
-    DOCUMENT_DOWNLOAD_API_HOST = getenv(
-        "DOCUMENT_DOWNLOAD_API_HOST", "http://localhost:7000"
-    )
-    DOCUMENT_DOWNLOAD_API_KEY = getenv("DOCUMENT_DOWNLOAD_API_KEY", "auth-token")
+    TEMPLATE_PREVIEW_API_HOST = getenv('TEMPLATE_PREVIEW_API_HOST', 'http://localhost:6013')
+    TEMPLATE_PREVIEW_API_KEY = getenv('TEMPLATE_PREVIEW_API_KEY', 'my-secret-key')
+
+    DOCUMENT_DOWNLOAD_API_HOST = getenv('DOCUMENT_DOWNLOAD_API_HOST', 'http://localhost:7000')
+    DOCUMENT_DOWNLOAD_API_KEY = getenv('DOCUMENT_DOWNLOAD_API_KEY', 'auth-token')
 
 
 def _s3_credentials_from_env(bucket_prefix):
     return {
-        "bucket": getenv(f"{bucket_prefix}_BUCKET_NAME"),
-        "access_key_id": getenv(f"{bucket_prefix}_AWS_ACCESS_KEY_ID"),
-        "secret_access_key": getenv(f"{bucket_prefix}_AWS_SECRET_ACCESS_KEY"),
-        "region": getenv(f"{bucket_prefix}_AWS_REGION"),
+        'bucket': getenv(f"{bucket_prefix}_BUCKET_NAME"),
+        'access_key_id': getenv(f"{bucket_prefix}_AWS_ACCESS_KEY_ID"),
+        'secret_access_key': getenv(f"{bucket_prefix}_AWS_SECRET_ACCESS_KEY"),
+        'region': getenv(f"{bucket_prefix}_AWS_REGION")
     }
 
 
@@ -313,52 +298,48 @@ class Development(Config):
     DEBUG = True
     NOTIFY_LOG_LEVEL = "DEBUG"
     SQLALCHEMY_ECHO = False
-    DVLA_EMAIL_ADDRESSES = ["success@simulator.amazonses.com"]
+    DVLA_EMAIL_ADDRESSES = ['success@simulator.amazonses.com']
 
     # Buckets
-    CSV_UPLOAD_BUCKET = _s3_credentials_from_env("CSV")
+    CSV_UPLOAD_BUCKET = _s3_credentials_from_env('CSV')
 
     # credential overrides
-    DANGEROUS_SALT = "development-notify-salt"
-    SECRET_KEY = (
-        "dev-notify-secret-key"  # nosec B105 - this is only used in development
-    )
-    INTERNAL_CLIENT_API_KEYS = {Config.ADMIN_CLIENT_ID: ["dev-notify-secret-key"]}
-    ALLOW_EXPIRED_API_TOKEN = getenv("ALLOW_EXPIRED_API_TOKEN", "0") == "1"
+    DANGEROUS_SALT = 'development-notify-salt'
+    SECRET_KEY = 'dev-notify-secret-key'  # nosec B105 - this is only used in development
+    INTERNAL_CLIENT_API_KEYS = {Config.ADMIN_CLIENT_ID: ['dev-notify-secret-key']}
+    ALLOW_EXPIRED_API_TOKEN = getenv('ALLOW_EXPIRED_API_TOKEN', '0') == '1'
 
 
 class Test(Development):
-    FROM_NUMBER = "testing"
+    FROM_NUMBER = 'testing'
     TESTING = True
     ANTIVIRUS_ENABLED = True
-    DVLA_EMAIL_ADDRESSES = [
-        "success@simulator.amazonses.com",
-        "success+2@simulator.amazonses.com",
-    ]
+    DVLA_EMAIL_ADDRESSES = ['success@simulator.amazonses.com', 'success+2@simulator.amazonses.com']
 
     HIGH_VOLUME_SERVICE = [
-        "941b6f9a-50d7-4742-8d50-f365ca74bf27",
-        "63f95b86-2d19-4497-b8b2-ccf25457df4e",
-        "7e5950cb-9954-41f5-8376-962b8c8555cf",
-        "10d1b9c9-0072-4fa9-ae1c-595e333841da",
+        '941b6f9a-50d7-4742-8d50-f365ca74bf27',
+        '63f95b86-2d19-4497-b8b2-ccf25457df4e',
+        '7e5950cb-9954-41f5-8376-962b8c8555cf',
+        '10d1b9c9-0072-4fa9-ae1c-595e333841da',
     ]
 
     # this is overriden in CI
-    SQLALCHEMY_DATABASE_URI = getenv("SQLALCHEMY_DATABASE_TEST_URI")
+    SQLALCHEMY_DATABASE_URI = getenv('SQLALCHEMY_DATABASE_TEST_URI')
 
     CELERY = {
         **Config.CELERY,
-        "broker_url": "you-forgot-to-mock-celery-in-your-tests://",
+        'broker_url': 'you-forgot-to-mock-celery-in-your-tests://'
     }
 
+    TEMPLATE_PREVIEW_API_HOST = 'http://localhost:9999'
+
 
 class Production(Config):
     # buckets
     CSV_UPLOAD_BUCKET = cloud_config.s3_credentials(
-        f"notify-api-csv-upload-bucket-{Config.NOTIFY_ENVIRONMENT}"
-    )
+        f"notify-api-csv-upload-bucket-{Config.NOTIFY_ENVIRONMENT}")
 
-    FROM_NUMBER = "Notify.gov"
+    FROM_NUMBER = 'US Notify'
     CRONITOR_ENABLED = True
 
 
@@ -371,10 +352,10 @@ class Demo(Production):
 
 
 configs = {
-    "development": Development,
-    "test": Test,
-    "staging": Staging,
-    "demo": Demo,
-    "sandbox": Staging,
-    "production": Production,
+    'development': Development,
+    'test': Test,
+    'staging': Staging,
+    'demo': Demo,
+    'sandbox': Staging,
+    'production': Production
 }

app/config_files/templates.json

@@ -1,314 +1,274 @@
 [
-  {
-    "id": "42a23d19-504e-49bb-a95e-4976baff4757",
-    "name": "Example text message template",
-    "type": "sms",
-    "subject": "",
-    "content": [
-      "Hi, I’m trying out Notify.gov. Today is ((day of week)) and my favorite color is ((color))."
-    ]
-  },
-  {
-    "id": "4f46df42-f795-4cc4-83bb-65ca312f49cc",
-    "name": "Notify invitation email",
-    "type": "email",
-    "subject": "((user_name)) has invited you to collaborate on ((service_name)) on Notify.gov",
-    "content": [
-      "((user_name)) has invited you to collaborate on ((service_name)) on Notify.gov.",
-      "",
-      "",
-      "Notify.gov makes it easy to keep people updated by helping you send text messages.",
-      "",
-      "",
-      "[Join Service](((url)))",
-      "If you’re new to Notify.gov you will first be directed to Login.gov create an account with us.",
-      "",
-      "",
-      "This invitation will stop working at midnight tomorrow. This is to keep ((service_name)) secure.",
-      "",
-      "",
-      "Notify.gov uses Login.gov to allow you to sign in safely. Login.gov is a secure sign in service to help you access participating government agencies' digital touchpoints."
-    ]
-  },
-  {
-    "id": "36fb0730-6259-4da1-8a80-c8de22ad4246",
-    "name": "Notify SMS verify code",
-    "type": "sms",
-    "subject": "",
-    "content": ["((verify_code)) is your Notify.gov authentication code."]
-  },
-  {
-    "id": "474e9242-823b-4f99-813d-ed392e7f1201",
-    "name": "Notify password reset email",
-    "type": "email",
-    "subject": "Reset your Notify.gov password",
-    "content": [
-      "Hi ((user_name)),",
-      "",
-      "",
-      "We received a request to reset your password on Notify.gov.",
-      "",
-      "",
-      "If you didn’t request this email, you can ignore it – your password has not been changed.",
-      "",
-      "",
-      "To reset your password, click this link:",
-      "",
-      "",
-      "((url))"
-    ]
-  },
-  {
-    "id": "299726d2-dba6-42b8-8209-30e1d66ea164",
-    "name": "Notify email verify code",
-    "type": "email",
-    "subject": "Sign in to Notify.gov",
-    "content": [
-      "Hi ((name)),",
-      "",
-      "",
-      "To sign in to Notify.gov please open this link:",
-      "",
-      "",
-      "((url))"
-    ]
-  },
-  {
-    "id": "ece42649-22a8-4d06-b87f-d52d5d3f0a27",
-    "name": "Notify email verification code",
-    "type": "email",
-    "subject": "Confirm Notify.gov registration",
-    "content": [
-      "Hi ((name)),",
-      "",
-      "",
-      "To complete your registration for Notify.gov please click the link below",
-      "",
-      "",
-      "[Join Service](((url)))"
-    ]
-  },
-  {
-    "id": "0880fbb1-a0c6-46f0-9a8e-36c986381ceb",
-    "name": "Your Notify.gov account",
-    "type": "email",
-    "subject": "Your Notify.gov account",
-    "content": [
-      "You already have a Notify.gov account with this email address.",
-      "",
-      "",
-      "Sign in here: ((signin_url))",
-      "",
-      "",
-      "If you’ve forgotten your password, you can reset it here: ((forgot_password_url))",
-      "",
-      "",
-      "",
-      "",
-      "If you didn’t try to register for a Notify.gov account recently, please let us know here: ((feedback_url))"
-    ]
-  },
-  {
-    "id": "eb4d9930-87ab-4aef-9bce-786762687884",
-    "name": "Confirm new email address",
-    "type": "email",
-    "subject": "Confirm your email address for Notify.gov",
-    "content": [
-      "Hi ((name)),",
-      "",
-      "",
-      "Click this link to confirm your new email address:",
-      "",
-      "",
-      "((url))",
-      "",
-      "",
-      "If you didn’t try to change the email address for your Notify.gov account, let us know here:",
-      "",
-      "",
-      "((feedback_url))"
-    ]
-  },
-  {
-    "id": "618185c6-3636-49cd-b7d2-6f6f5eb3bdde",
-    "name": "Automated \"You’re now live\" message",
-    "type": "email",
-    "subject": "((service name)) is now live on Notify.gov",
-    "content": [
-      "Hi ((name)),",
-      "",
-      "",
-      "Congratulations, we have received and reviewed your Go Live submission form. ((service name)) is now live on Notify.gov. With your Notify account you now can send messages for the use case outlined in your Go Live submission form.",
-      "",
-      "",
-      "If you have any questions or experience any technical issues, please contact your pilot lead.",
-      "",
-      "",
-      "Thanks",
-      "",
-      "Notify.gov team"
-    ]
-  },
-  {
-    "id": "203566f0-d835-47c5-aa06-932439c86573",
-    "name": "Notify organization invitation email",
-    "type": "email",
-    "subject": "((user_name)) has invited you to collaborate on ((organization_name)) on Notify.gov",
-    "content": [
-      "((user_name)) has invited you to collaborate on ((organization_name)) on Notify.gov.",
-      "",
-      "",
-      "Notify.gov makes it easy to keep people updated by helping you send text messages and emails.",
-      "",
-      "",
-      "Open this link to create an account on Notify.gov:",
-      "",
-      "[Join Organization](((url)))",
-      "",
-      "",
-      "This invitation will stop working at midnight tomorrow. This is to keep ((organization_name)) secure."
-    ]
-  },
-  {
-    "id": "c73f1d71-4049-46d5-a647-d013bdeca3f0",
-    "name": "Email address changed by service manager",
-    "type": "email",
-    "subject": "Your Notify.gov email address has changed",
-    "content": [
-      "Dear ((name)),",
-      "",
-      "",
-      "((servicemanagername)) changed your Notify account email address to:",
-      "",
-      "",
-      "((email address))",
-      "",
-      "",
-      "You’ll need to use this email address next time you sign in.",
-      "",
-      "",
-      "Thanks",
-      "",
-      "",
-      "Notify.gov team"
-    ]
-  },
-  {
-    "id": "8a31520f-4751-4789-8ea1-fe54496725eb",
-    "name": "Phone number changed by service manager",
-    "type": "sms",
-    "subject": "",
-    "content": [
-      "Your mobile number was changed by ((servicemanagername)). Next time you sign in, your Notify.gov authentication code will be sent to this phone."
-    ]
-  },
-  {
-    "id": "a42f1d17-9404-46d5-a647-d013bdfca3e1",
-    "name": "Verify email reply-to address for a service",
-    "type": "email",
-    "subject": "Your Notify.gov reply-to email address",
-    "content": [
-      "Hi,",
-      "",
-      "",
-      "This address has been provided as a reply-to email address for a Notify.gov account.",
-      "",
-      "Any replies from users to emails they receive through Notify.gov will come back to this email address.",
-      "",
-      "",
-      "This is just a quick check to make sure the address is valid.",
-      "",
-      "",
-      "No need to reply.",
-      "",
-      "",
-      "Thanks",
-      "",
-      "",
-      "Notify.gov team"
-    ]
-  },
-  {
-    "id": "4fd2e43c-309b-4e50-8fb8-1955852d9d71",
-    "name": "MOU Signed By Receipt",
-    "type": "email",
-    "subject": "You’ve accepted the Notify.gov data sharing and financial agreement",
-    "content": [
-      "Hi ((signed_by_name)),",
-      "",
-      "((org_name)) has accepted the Notify.gov data sharing and financial agreement. ",
-      "",
-      "If you need another copy of the agreement you can download it here: ((mou_link))",
-      "",
-      "",
-      "Thanks,",
-      "Notify.gov team"
-    ]
-  },
-  {
-    "id": "c20206d5-bf03-4002-9a90-37d5032d9e84",
-    "name": "MOU Signed On Behalf Of Receipt - Signed by",
-    "type": "email",
-    "subject": "You’ve accepted the Notify.gov data sharing and financial agreement",
-    "content": [
-      "Hi ((signed_by_name)),",
-      "",
-      "((org_name)) has accepted the Notify.gov data sharing and financial agreement. We’ve emailed ((on_behalf_of_name)) to let them know too.",
-      "",
-      "If you need another copy of the agreement you can download it here: ((mou_link))",
-      "",
-      "",
-      "Thanks,",
-      "Notify.gov team"
-    ]
-  },
-  {
-    "id": "522b6657-5ca5-4368-a294-6b527703bd0b",
-    "name": "MOU Signed On Behalf Of Receipt - On Behalf Of",
-    "type": "email",
-    "subject": "((org_name)) has accepted the Notify.gov data sharing and financial agreement",
-    "content": [
-      "Hi ((on_behalf_of_name)),",
-      "",
-      "((signed_by_name)) has accepted the Notify.gov data sharing and financial agreement on your behalf, for ((org_name)).",
-      "",
-      "Notify.gov lets teams in the public sector send text messages and emails. It’s built and run by a team in the TTS Public Benefits Studio (part of GSA).",
-      "",
-      "If you need another copy of the agreement you can download it here: ((mou_link))",
-      "",
-      "",
-      "Thanks,",
-      "Notify.gov team"
-    ]
-  },
-  {
-    "id": "d0e66c4c-0c50-43f0-94f5-f85b613202d4",
-    "name": "MOU Signed Notify Team Alert",
-    "type": "email",
-    "subject": "Someone signed an MOU for an org on Notify",
-    "content": [
-      "What’s up Notifiers,",
-      "",
-      "((signed_by_name)) just accepted the data sharing and financial agreement for ((org_name)).",
-      "",
-      "See how ((org_name)) is using Notify here: ((org_dashboard_link))"
-    ]
-  },
-  {
-    "id": "11fad854-fd38-4a7c-bd17-805fb13dfc12",
-    "name": "Notify daily letter volumes",
-    "type": "email",
-    "subject": "Notify letter volume for ((date)): ((total_volume)) letters, ((total_sheets)) sheets",
-    "content": [
-      "((total_volume)) letters (((total_sheets)) sheets) sent via Notify are coming in today’s batch. These include: ",
-      "",
-      "((first_class_volume)) first class letters (((first_class_sheets)) sheets).",
-      "((second_class_volume)) second class letters (((second_class_sheets)) sheets).",
-      "((international_volume)) international letters (((international_sheets)) sheets).",
-      "",
-      "Thanks",
-      "",
-      "Notify.gov team"
-    ]
-  }
+    {
+        "id": "42a23d19-504e-49bb-a95e-4976baff4757",
+        "name": "Example text message template",
+        "type": "sms",
+        "subject":  "",
+        "content": ["Hi, I’m trying out U.S. Notify. Today is ((day of week)) and my favorite color is ((color))."]
+    },
+    {
+        "id": "4f46df42-f795-4cc4-83bb-65ca312f49cc",
+        "name": "Notify invitation email",
+        "type": "email",
+        "subject":  "((user_name)) has invited you to collaborate on ((service_name)) on U.S. Notify",
+        "content": ["((user_name)) has invited you to collaborate on ((service_name)) on U.S. Notify.",
+            "",
+            "",
+            "U.S. Notify makes it easy to keep people updated by helping you send text messages and emails.",
+            "",
+            "",
+            "Click this link to create an account on U.S. Notify:",
+            "",
+            "((url))",
+            "",
+            "",
+            "This invitation will stop working at midnight tomorrow. This is to keep ((service_name)) secure."]
+    },
+    {
+        "id": "36fb0730-6259-4da1-8a80-c8de22ad4246",
+        "name": "Notify SMS verify code",
+        "type": "sms",
+        "subject": "",
+        "content": ["((verify_code)) is your U.S. Notify authentication code"]
+    },
+    {
+        "id": "474e9242-823b-4f99-813d-ed392e7f1201",
+        "name": "Notify password reset email",
+        "type": "email",
+        "subject": "Reset your U.S. Notify password",
+        "content": ["Hi ((user_name)),",
+            "",
+            "",
+            "We received a request to reset your password on U.S. Notify.",
+            "",
+            "",
+            "If you didn’t request this email, you can ignore it – your password has not been changed.",
+            "",
+            "",
+            "To reset your password, click this link:","","","((url))"]
+    },
+    {
+        "id": "299726d2-dba6-42b8-8209-30e1d66ea164",
+        "name": "Notify email verify code",
+        "type": "email",
+        "subject": "Sign in to U.S. Notify",
+        "content": ["Hi ((name)),",
+        "",
+        "",
+        "To sign in to U.S. Notify please open this link:",
+        "",
+        "",
+        "((url))"]
+    },
+    {
+        "id": "ece42649-22a8-4d06-b87f-d52d5d3f0a27",
+        "name": "Notify email verification code",
+        "type": "email",
+        "subject": "Confirm U.S. Notify registration",
+        "content": ["Hi ((name)),",
+        "",
+        "",
+        "To complete your registration for U.S. Notify please click the link below",
+        "",
+        "",
+        "((url))"]
+    },
+    {
+        "id": "0880fbb1-a0c6-46f0-9a8e-36c986381ceb",
+        "name": "Your U.S. Notify account",
+        "type": "email",
+        "subject": "Your U.S. Notify account",
+        "content": ["You already have a U.S. Notify account with this email address.",
+        "",
+        "",
+        "Sign in here: ((signin_url))",
+        "",
+        "",
+        "If you’ve forgotten your password, you can reset it here: ((forgot_password_url))",
+        "",
+        "",
+        "",
+        "",
+        "If you didn’t try to register for a U.S. Notify account recently, please let us know here: ((feedback_url))"]
+    },
+    {
+        "id": "eb4d9930-87ab-4aef-9bce-786762687884",
+        "name": "Confirm new email address",
+        "type": "email",
+        "subject": "Confirm your email address for U.S. Notify",
+        "content": ["Hi ((name)),","","","Click this link to confirm your new email address:",
+            "",
+            "",
+            "((url))",
+            "",
+            "",
+            "If you didn’t try to change the email address for your U.S. Notify account, let us know here:",
+            "",
+            "",
+            "((feedback_url))"]
+    },
+    {
+        "id": "618185c6-3636-49cd-b7d2-6f6f5eb3bdde",
+        "name": "Automated \"You’re now live\" message",
+        "type": "email",
+        "subject": "((service name)) is now live on U.S. Notify",
+        "content": ["Hi ((name)),",
+        "",
+        "",
+        "((service name)) is now live on U.S. Notify.",
+        "",
+        "",
+        "You can send up to ((message limit)) messages per day.",
+        "",
+        "",
+        "As a live service, you’ll need to know who to contact if you have a question, or something goes wrong.",
+        "",
+        "",
+        "If our system status page shows a problem, then we’ve been alerted and are working on it – you don’t need to contact us.",
+        "",
+        "",
+        "#Problems or questions out of hours",
+        "",
+        "",
+        "We offer out of hours support for emergencies.",
+        "",
+        "",
+        "It’s only an emergency if:",
+        "",
+        "*   no one in your team can log in",
+        "",
+        "*   a ‘technical difficulties’ error appears when you try to upload a file",
+        "",
+        "*   a 500 response code appears when you try to send messages using the API",
+        "",
+        "",
+        "If you have one of these emergencies, email details to:",
+        "",
+        "notify-support@gsa.gov",
+        "",
+        "",
+        "^Only use this email address for out of hours emergencies. Don’t share this address with people outside of your team.",
+        "",
+        "",
+        "We’ll get back to you within 30 minutes and give you hourly updates until the problem’s fixed.",
+        "",
+        "",
+        "For non-emergency problems or questions, use our support page and we’ll reply in office hours.",
+        "",
+        "",
+        "Thanks",
+        "",
+        "U.S. Notify team"]
+    },
+    {
+        "id": "203566f0-d835-47c5-aa06-932439c86573",
+        "name": "Notify organization invitation email",
+        "type": "email",
+        "subject": "((user_name)) has invited you to collaborate on ((organization_name)) on U.S. Notify",
+        "content": ["((user_name)) has invited you to collaborate on ((organization_name)) on U.S. Notify.","","","U.S. Notify makes it easy to keep people updated by helping you send text messages and emails.","","","Open this link to create an account on U.S. Notify:","","((url))","","","This invitation will stop working at midnight tomorrow. This is to keep ((organization_name)) secure."]
+    },
+    {
+        "id": "c73f1d71-4049-46d5-a647-d013bdeca3f0",
+        "name": "Email address changed by service manager",
+        "type": "email",
+        "subject": "Your U.S. Notify email address has changed",
+        "content":  ["Dear ((name)),","","","((servicemanagername)) changed your Notify account email address to:","","","((email address))","","","You’ll need to use this email address next time you sign in.","","","Thanks","","","U.S. Notify team"]
+    },
+    {
+        "id": "8a31520f-4751-4789-8ea1-fe54496725eb",
+        "name": "Phone number changed by service manager",
+        "type": "sms",
+        "subject": "",
+        "content": ["Your mobile number was changed by ((servicemanagername)). Next time you sign in, your U.S. Notify authentication code will be sent to this phone."]
+    },
+    {
+        "id": "a42f1d17-9404-46d5-a647-d013bdfca3e1",
+        "name": "Verify email reply-to address for a service",
+        "type": "email",
+        "subject": "Your U.S. Notify reply-to email address",
+        "content": ["Hi,","","","This address has been provided as a reply-to email address for a U.S. Notify account.","","Any replies from users to emails they receive through U.S. Notify will come back to this email address.","","","This is just a quick check to make sure the address is valid.","","","No need to reply.","","","Thanks","","","U.S. Notify team"]
+    },
+    {
+        "id": "4fd2e43c-309b-4e50-8fb8-1955852d9d71",
+        "name": "MOU Signed By Receipt",
+        "type": "email",
+        "subject": "You’ve accepted the U.S. Notify data sharing and financial agreement",
+        "content": [
+            "Hi ((signed_by_name)),",
+            "",
+            "((org_name)) has accepted the U.S. Notify data sharing and financial agreement. ",
+            "",
+            "If you need another copy of the agreement you can download it here: ((mou_link))",
+            "",
+            "",
+            "Thanks,",
+            "U.S. Notify team"
+        ]
+    },
+    {
+        "id": "c20206d5-bf03-4002-9a90-37d5032d9e84",
+        "name": "MOU Signed On Behalf Of Receipt - Signed by",
+        "type": "email",
+        "subject": "You’ve accepted the U.S. Notify data sharing and financial agreement",
+        "content": [
+            "Hi ((signed_by_name)),",
+            "",
+            "((org_name)) has accepted the U.S. Notify data sharing and financial agreement. We’ve emailed ((on_behalf_of_name)) to let them know too.",
+            "",
+            "If you need another copy of the agreement you can download it here: ((mou_link))",
+            "",
+            "",
+            "Thanks,",
+            "U.S. Notify team"
+        ]
+    },
+    {
+        "id": "522b6657-5ca5-4368-a294-6b527703bd0b",
+        "name": "MOU Signed On Behalf Of Receipt - On Behalf Of",
+        "type": "email",
+        "subject": "((org_name)) has accepted the U.S. Notify data sharing and financial agreement",
+        "content": [
+            "Hi ((on_behalf_of_name)),",
+            "",
+            "((signed_by_name)) has accepted the U.S. Notify data sharing and financial agreement on your behalf, for ((org_name)).",
+            "",
+            "U.S. Notify lets teams in the public sector send text messages and emails. It’s built and run by a team in the TTS Public Benefits Studio (part of GSA).",
+            "",
+            "If you need another copy of the agreement you can download it here: ((mou_link))",
+            "",
+            "",
+            "Thanks,",
+            "U.S. Notify team"
+        ]
+    },
+    {
+        "id": "d0e66c4c-0c50-43f0-94f5-f85b613202d4",
+        "name": "MOU Signed Notify Team Alert",
+        "type": "email",
+        "subject": "Someone signed an MOU for an org on Notify",
+        "content": [
+            "What’s up Notifiers,",
+            "",
+            "((signed_by_name)) just accepted the data sharing and financial agreement for ((org_name)).",
+            "",
+            "See how ((org_name)) is using Notify here: ((org_dashboard_link))"
+        ]
+    },
+    {
+        "id": "11fad854-fd38-4a7c-bd17-805fb13dfc12",
+        "name": "Notify daily letter volumes",
+        "type": "email",
+        "subject": "Notify letter volume for ((date)): ((total_volume)) letters, ((total_sheets)) sheets",
+        "content": [
+            "((total_volume)) letters (((total_sheets)) sheets) sent via Notify are coming in today’s batch. These include: ",
+            "",
+            "((first_class_volume)) first class letters (((first_class_sheets)) sheets).",
+            "((second_class_volume)) second class letters (((second_class_sheets)) sheets).",
+            "((international_volume)) international letters (((international_sheets)) sheets).",
+            "",
+            "Thanks",
+            "",
+            "U.S. Notify team"
+        ]
+    }
 ]

app/cronitor.py

@@ -7,54 +7,48 @@ from flask import current_app
 def cronitor(task_name):
     def decorator(func):
         def ping_cronitor(command):
-            if not current_app.config["CRONITOR_ENABLED"]:
+            if not current_app.config['CRONITOR_ENABLED']:
                 return
 
             # it's useful to have a log that a periodic task has started in case it
             # get stuck without generating any other logs - we know it got this far
-            current_app.logger.info(f"Pinging Cronitor for Celery task {task_name}")
+            current_app.logger.info(f'Pinging Cronitor for Celery task {task_name}')
 
-            task_slug = current_app.config["CRONITOR_KEYS"].get(task_name)
+            task_slug = current_app.config['CRONITOR_KEYS'].get(task_name)
             if not task_slug:
                 current_app.logger.error(
-                    "Cronitor enabled but task_name {} not found in environment".format(
-                        task_name
-                    )
+                    'Cronitor enabled but task_name {} not found in environment'.format(task_name)
                 )
                 return
 
-            if command not in {"run", "complete", "fail"}:
-                raise ValueError(
-                    "command {} not a valid cronitor command".format(command)
-                )
+            if command not in {'run', 'complete', 'fail'}:
+                raise ValueError('command {} not a valid cronitor command'.format(command))
 
             try:
                 resp = requests.get(
-                    "https://cronitor.link/{}/{}".format(task_slug, command),
+                    'https://cronitor.link/{}/{}'.format(task_slug, command),
                     # cronitor limits msg to 1000 characters
                     params={
-                        "host": current_app.config["API_HOST_NAME"],
-                    },
+                        'host': current_app.config['API_HOST_NAME'],
+                    }
                 )
                 resp.raise_for_status()
             except requests.RequestException as e:
-                current_app.logger.warning(
-                    "Cronitor API failed for task {} due to {}".format(
-                        task_name, repr(e)
-                    )
-                )
+                current_app.logger.warning('Cronitor API failed for task {} due to {}'.format(
+                    task_name,
+                    repr(e)
+                ))
 
         @wraps(func)
         def inner_decorator(*args, **kwargs):
-            ping_cronitor("run")
-            status = "fail"
+            ping_cronitor('run')
+            status = 'fail'
             try:
                 ret = func(*args, **kwargs)
-                status = "complete"
+                status = 'complete'
                 return ret
             finally:
                 ping_cronitor(status)
 
         return inner_decorator
-
     return decorator

app/dao/__init__.py

@@ -9,6 +9,7 @@ class DAOException(SQLAlchemyError):
 
 
 class DAOClass(object):
+
     class Meta:
         model = None
 
@@ -17,6 +18,13 @@ class DAOClass(object):
         if _commit:
             db.session.commit()
 
+    def update_instance(self, inst, update_dict, _commit=True):
+        # Make sure the id is not included in the update_dict
+        update_dict.pop('id')
+        self.Meta.model.query.filter_by(id=inst.id).update(update_dict)
+        if _commit:
+            db.session.commit()
+
     def delete_instance(self, inst, _commit=True):
         db.session.delete(inst)
         if _commit:

app/dao/annual_billing_dao.py

@@ -3,84 +3,73 @@ from flask import current_app
 from app import db
 from app.dao.dao_utils import autocommit
 from app.dao.date_util import get_current_calendar_year_start_year
-from app.enums import OrganizationType
 from app.models import AnnualBilling
 
 
 @autocommit
-def dao_create_or_update_annual_billing_for_year(
-    service_id, free_sms_fragment_limit, financial_year_start
-):
+def dao_create_or_update_annual_billing_for_year(service_id, free_sms_fragment_limit, financial_year_start):
     result = dao_get_free_sms_fragment_limit_for_year(service_id, financial_year_start)
 
     if result:
         result.free_sms_fragment_limit = free_sms_fragment_limit
     else:
-        result = AnnualBilling(
-            service_id=service_id,
-            financial_year_start=financial_year_start,
-            free_sms_fragment_limit=free_sms_fragment_limit,
-        )
+        result = AnnualBilling(service_id=service_id, financial_year_start=financial_year_start,
+                               free_sms_fragment_limit=free_sms_fragment_limit)
     db.session.add(result)
     return result
 
 
 def dao_get_annual_billing(service_id):
-    return (
-        AnnualBilling.query.filter_by(
-            service_id=service_id,
-        )
-        .order_by(AnnualBilling.financial_year_start)
-        .all()
-    )
+    return AnnualBilling.query.filter_by(
+        service_id=service_id,
+    ).order_by(AnnualBilling.financial_year_start).all()
 
 
 @autocommit
-def dao_update_annual_billing_for_future_years(
-    service_id, free_sms_fragment_limit, financial_year_start
-):
+def dao_update_annual_billing_for_future_years(service_id, free_sms_fragment_limit, financial_year_start):
     AnnualBilling.query.filter(
         AnnualBilling.service_id == service_id,
-        AnnualBilling.financial_year_start > financial_year_start,
-    ).update({"free_sms_fragment_limit": free_sms_fragment_limit})
+        AnnualBilling.financial_year_start > financial_year_start
+    ).update(
+        {'free_sms_fragment_limit': free_sms_fragment_limit}
+    )
 
 
 def dao_get_free_sms_fragment_limit_for_year(service_id, financial_year_start=None):
+
     if not financial_year_start:
         financial_year_start = get_current_calendar_year_start_year()
 
     return AnnualBilling.query.filter_by(
-        service_id=service_id, financial_year_start=financial_year_start
+        service_id=service_id,
+        financial_year_start=financial_year_start
     ).first()
 
 
 def dao_get_all_free_sms_fragment_limit(service_id):
-    return (
-        AnnualBilling.query.filter_by(
-            service_id=service_id,
-        )
-        .order_by(AnnualBilling.financial_year_start)
-        .all()
-    )
+
+    return AnnualBilling.query.filter_by(
+        service_id=service_id,
+    ).order_by(AnnualBilling.financial_year_start).all()
 
 
 def set_default_free_allowance_for_service(service, year_start=None):
     default_free_sms_fragment_limits = {
-        OrganizationType.FEDERAL: {
+        'federal': {
             2020: 250_000,
             2021: 150_000,
             2022: 40_000,
         },
-        OrganizationType.STATE: {
+        'state': {
             2020: 250_000,
             2021: 150_000,
             2022: 40_000,
         },
-        OrganizationType.OTHER: {
+        'other': {
             2020: 250_000,
             2021: 150_000,
             2022: 40_000,
-        },
+        }
     }
     if not year_start:
         year_start = get_current_calendar_year_start_year()
@@ -90,18 +79,14 @@ def set_default_free_allowance_for_service(service, year_start=None):
     if year_start > 2022:
         year_start = 2022
     if service.organization_type:
-        free_allowance = default_free_sms_fragment_limits[service.organization_type][
-            year_start
-        ]
+        free_allowance = default_free_sms_fragment_limits[service.organization_type][year_start]
     else:
-        current_app.logger.info(
-            f"no organization type for service {service.id}. Using other default of "
-            f"{default_free_sms_fragment_limits['other'][year_start]}"
-        )
-        free_allowance = default_free_sms_fragment_limits[OrganizationType.OTHER][
-            year_start
-        ]
+        current_app.logger.info(f"no organization type for service {service.id}. Using other default of "
+                                f"{default_free_sms_fragment_limits['other'][year_start]}")
+        free_allowance = default_free_sms_fragment_limits['other'][year_start]
 
     return dao_create_or_update_annual_billing_for_year(
-        service.id, free_allowance, year_start
+        service.id,
+        free_allowance,
+        year_start
     )

app/dao/api_key_dao.py

@@ -1,21 +1,18 @@
 import uuid
-from datetime import timedelta
+from datetime import datetime, timedelta
 
 from sqlalchemy import func, or_
 
 from app import db
 from app.dao.dao_utils import autocommit, version_class
 from app.models import ApiKey
-from app.utils import utc_now
 
 
 @autocommit
 @version_class(ApiKey)
 def save_model_api_key(api_key):
     if not api_key.id:
-        api_key.id = (
-            uuid.uuid4()
-        )  # must be set now so version history model can use same id
+        api_key.id = uuid.uuid4()  # must be set now so version history model can use same id
     api_key.secret = uuid.uuid4()
     db.session.add(api_key)
 
@@ -24,22 +21,17 @@ def save_model_api_key(api_key):
 @version_class(ApiKey)
 def expire_api_key(service_id, api_key_id):
     api_key = ApiKey.query.filter_by(id=api_key_id, service_id=service_id).one()
-    api_key.expiry_date = utc_now()
+    api_key.expiry_date = datetime.utcnow()
     db.session.add(api_key)
 
 
 def get_model_api_keys(service_id, id=None):
     if id:
-        return ApiKey.query.filter_by(
-            id=id, service_id=service_id, expiry_date=None
-        ).one()
-    seven_days_ago = utc_now() - timedelta(days=7)
+        return ApiKey.query.filter_by(id=id, service_id=service_id, expiry_date=None).one()
+    seven_days_ago = datetime.utcnow() - timedelta(days=7)
     return ApiKey.query.filter(
-        or_(
-            ApiKey.expiry_date == None,  # noqa
-            func.date(ApiKey.expiry_date) > seven_days_ago,  # noqa
-        ),
-        ApiKey.service_id == service_id,
+        or_(ApiKey.expiry_date == None, func.date(ApiKey.expiry_date) > seven_days_ago),  # noqa
+        ApiKey.service_id == service_id
     ).all()
 
 

app/dao/complaint_dao.py

@@ -15,23 +15,20 @@ def save_complaint(complaint):
 
 
 def fetch_paginated_complaints(page=1):
-    return Complaint.query.order_by(desc(Complaint.created_at)).paginate(
-        page=page, per_page=current_app.config["PAGE_SIZE"]
+    return Complaint.query.order_by(
+        desc(Complaint.created_at)
+    ).paginate(
+        page=page,
+        per_page=current_app.config['PAGE_SIZE']
     )
 
 
 def fetch_complaints_by_service(service_id):
-    return (
-        Complaint.query.filter_by(service_id=service_id)
-        .order_by(desc(Complaint.created_at))
-        .all()
-    )
+    return Complaint.query.filter_by(service_id=service_id).order_by(desc(Complaint.created_at)).all()
 
 
 def fetch_count_of_complaints(start_date, end_date):
     start_date = get_midnight_in_utc(start_date)
     end_date = get_midnight_in_utc(end_date + timedelta(days=1))
 
-    return Complaint.query.filter(
-        Complaint.created_at >= start_date, Complaint.created_at < end_date
-    ).count()
+    return Complaint.query.filter(Complaint.created_at >= start_date, Complaint.created_at < end_date).count()

app/dao/dao_utils.py

@@ -12,14 +12,13 @@ def autocommit(func):
         try:
             res = func(*args, **kwargs)
 
-            if not db.session().in_nested_transaction():
+            if not db.session.registry().transaction.nested:
                 db.session.commit()
 
             return res
         except Exception:
             db.session.rollback()
             raise
-
     return commit_or_rollback
 
 
@@ -30,14 +29,15 @@ def transaction():
         yield
         db.session.commit()
 
-        if not db.session().in_nested_transaction():
+        if not db.session.registry().transaction.nested:
             db.session.commit()
     except Exception:
         db.session.rollback()
         raise
 
 
-class VersionOptions:
+class VersionOptions():
+
     def __init__(self, model_class, history_class=None, must_write_history=True):
         self.model_class = model_class
         self.history_class = history_class
@@ -45,34 +45,38 @@ class VersionOptions:
 
 
 def version_class(*version_options):
+
     if len(version_options) == 1 and not isinstance(version_options[0], VersionOptions):
         version_options = (VersionOptions(version_options[0]),)
 
     def versioned(func):
         @wraps(func)
         def record_version(*args, **kwargs):
+
             func(*args, **kwargs)
 
             session_objects = []
 
             for version_option in version_options:
                 tmp_session_objects = [
-                    (session_object, version_option.history_class)
+                    (
+                        session_object, version_option.history_class
+                    )
                     for session_object in itertools.chain(
                         db.session.new, db.session.dirty
                     )
-                    if isinstance(session_object, version_option.model_class)
+                    if isinstance(
+                        session_object, version_option.model_class
+                    )
                 ]
 
                 if tmp_session_objects == [] and version_option.must_write_history:
-                    raise RuntimeError(
-                        (
-                            "Can't record history for {} "
-                            "(something in your code has casued the database to "
-                            "flush the session early so there's nothing to "
-                            "copy into the history table)"
-                        ).format(version_option.model_class.__name__)
-                    )
+                    raise RuntimeError((
+                        'Can\'t record history for {} '
+                        '(something in your code has casued the database to '
+                        'flush the session early so there\'s nothing to '
+                        'copy into the history table)'
+                    ).format(version_option.model_class.__name__))
 
                 session_objects += tmp_session_objects
 
@@ -82,7 +86,6 @@ def version_class(*version_options):
                 )
 
         return record_version
-
     return versioned
 
 

app/dao/date_util.py

@@ -1,12 +1,13 @@
-import calendar
 from datetime import date, datetime, time, timedelta
 
-from app.utils import utc_now
-
 
 def get_months_for_financial_year(year):
     return [
-        month for month in (get_months_for_year(1, 13, year)) if month < datetime.now()
+        month for month in (
+            get_months_for_year(4, 13, year)
+            + get_months_for_year(1, 4, year + 1)
+        )
+        if month < datetime.now()
     ]
 
 
@@ -21,12 +22,15 @@ def get_calendar_year(year):
 def get_calendar_year_dates(year):
     year_start_datetime, year_end_datetime = get_calendar_year(year)
 
-    return (year_start_datetime.date(), year_end_datetime.date())
+    return (
+        year_start_datetime.date(),
+        year_end_datetime.date()
+    )
 
 
 def get_current_calendar_year():
-    now = utc_now()
-    current_year = int(now.strftime("%Y"))
+    now = datetime.utcnow()
+    current_year = int(now.strftime('%Y'))
     year = current_year
     return get_calendar_year(year)
 
@@ -37,12 +41,11 @@ def get_new_years(year):
 
 def get_month_start_and_end_date_in_utc(month_year):
     """
-    This function return the start and date of the month_year as UTC,
-    :param month_year: the datetime to calculate the start and end date for that month
-    :return: start_date, end_date, month
+     This function return the start and date of the month_year as UTC,
+     :param month_year: the datetime to calculate the start and end date for that month
+     :return: start_date, end_date, month
     """
     import calendar
-
     _, num_days = calendar.monthrange(month_year.year, month_year.month)
     first_day = datetime(month_year.year, month_year.month, 1, 0, 0, 0)
     last_day = datetime(month_year.year, month_year.month, num_days, 23, 59, 59, 99999)
@@ -59,37 +62,11 @@ def get_current_calendar_year_start_year():
 
 
 def get_calendar_year_for_datetime(start_date):
-    if isinstance(start_date, date):
+    if type(start_date) == date:
         start_date = datetime.combine(start_date, time.min)
 
-    year = int(start_date.strftime("%Y"))
+    year = int(start_date.strftime('%Y'))
     if start_date < get_new_years(year):
         return year - 1
     else:
         return year
-
-
-def get_number_of_days_for_month(year, month):
-    return calendar.monthrange(year, month)[1]
-
-
-def generate_date_range(start_date, end_date=None, days=0):
-    if end_date:
-        current_date = start_date
-        while current_date <= end_date:
-            try:
-                yield current_date.date()
-            except ValueError:
-                pass
-            current_date += timedelta(days=1)
-    elif days > 0:
-        end_date = start_date + timedelta(days=days)
-        current_date = start_date
-        while current_date < end_date:
-            try:
-                yield current_date.date()
-            except ValueError:
-                pass
-            current_date += timedelta(days=1)
-    else:
-        return "An end_date or number of days must be specified"

app/dao/fact_notification_status_dao.py

@@ -1,4 +1,4 @@
-from datetime import timedelta
+from datetime import datetime, timedelta
 
 from sqlalchemy import Date, case, func
 from sqlalchemy.dialects.postgresql import insert
@@ -7,8 +7,20 @@ from sqlalchemy.types import DateTime, Integer
 
 from app import db
 from app.dao.dao_utils import autocommit
-from app.enums import KeyType, NotificationStatus, NotificationType
 from app.models import (
+    KEY_TYPE_NORMAL,
+    KEY_TYPE_TEAM,
+    KEY_TYPE_TEST,
+    NOTIFICATION_CANCELLED,
+    NOTIFICATION_CREATED,
+    NOTIFICATION_DELIVERED,
+    NOTIFICATION_FAILED,
+    NOTIFICATION_PENDING,
+    NOTIFICATION_PERMANENT_FAILURE,
+    NOTIFICATION_SENDING,
+    NOTIFICATION_SENT,
+    NOTIFICATION_TECHNICAL_FAILURE,
+    NOTIFICATION_TEMPORARY_FAILURE,
     FactNotificationStatus,
     Notification,
     NotificationAllTimeView,
@@ -19,7 +31,6 @@ from app.utils import (
     get_midnight_in_utc,
     get_month_from_utc_column,
     midnight_n_days_ago,
-    utc_now,
 )
 
 
@@ -35,33 +46,27 @@ def update_fact_notification_status(process_day, notification_type, service_id):
         FactNotificationStatus.service_id == service_id,
     ).delete()
 
-    query = (
-        db.session.query(
-            literal(process_day).label("process_day"),
-            NotificationAllTimeView.template_id,
-            literal(service_id).label("service_id"),
-            func.coalesce(
-                NotificationAllTimeView.job_id, "00000000-0000-0000-0000-000000000000"
-            ).label("job_id"),
-            literal(notification_type).label("notification_type"),
-            NotificationAllTimeView.key_type,
-            NotificationAllTimeView.status,
-            func.count().label("notification_count"),
-        )
-        .filter(
-            NotificationAllTimeView.created_at >= start_date,
-            NotificationAllTimeView.created_at < end_date,
-            NotificationAllTimeView.notification_type == notification_type,
-            NotificationAllTimeView.service_id == service_id,
-            NotificationAllTimeView.key_type.in_((KeyType.NORMAL, KeyType.TEAM)),
-        )
-        .group_by(
-            NotificationAllTimeView.template_id,
-            NotificationAllTimeView.template_id,
-            "job_id",
-            NotificationAllTimeView.key_type,
-            NotificationAllTimeView.status,
-        )
+    query = db.session.query(
+        literal(process_day).label("process_day"),
+        NotificationAllTimeView.template_id,
+        literal(service_id).label("service_id"),
+        func.coalesce(NotificationAllTimeView.job_id, '00000000-0000-0000-0000-000000000000').label('job_id'),
+        literal(notification_type).label("notification_type"),
+        NotificationAllTimeView.key_type,
+        NotificationAllTimeView.status,
+        func.count().label('notification_count')
+    ).filter(
+        NotificationAllTimeView.created_at >= start_date,
+        NotificationAllTimeView.created_at < end_date,
+        NotificationAllTimeView.notification_type == notification_type,
+        NotificationAllTimeView.service_id == service_id,
+        NotificationAllTimeView.key_type.in_((KEY_TYPE_NORMAL, KEY_TYPE_TEAM)),
+    ).group_by(
+        NotificationAllTimeView.template_id,
+        NotificationAllTimeView.template_id,
+        'job_id',
+        NotificationAllTimeView.key_type,
+        NotificationAllTimeView.status
     )
 
     db.session.connection().execute(
@@ -74,113 +79,89 @@ def update_fact_notification_status(process_day, notification_type, service_id):
                 FactNotificationStatus.notification_type,
                 FactNotificationStatus.key_type,
                 FactNotificationStatus.notification_status,
-                FactNotificationStatus.notification_count,
+                FactNotificationStatus.notification_count
             ],
-            query,
+            query
         )
     )
 
 
 def fetch_notification_status_for_service_by_month(start_date, end_date, service_id):
-    return (
-        db.session.query(
-            func.date_trunc("month", NotificationAllTimeView.created_at).label("month"),
-            NotificationAllTimeView.notification_type,
-            NotificationAllTimeView.status.label("notification_status"),
-            func.count(NotificationAllTimeView.id).label("count"),
-        )
-        .filter(
-            NotificationAllTimeView.service_id == service_id,
-            NotificationAllTimeView.created_at >= start_date,
-            NotificationAllTimeView.created_at < end_date,
-            NotificationAllTimeView.key_type != KeyType.TEST,
-        )
-        .group_by(
-            func.date_trunc("month", NotificationAllTimeView.created_at).label("month"),
-            NotificationAllTimeView.notification_type,
-            NotificationAllTimeView.status,
-        )
-        .all()
-    )
-
-
-def fetch_notification_status_for_service_for_day(fetch_day, service_id):
-    return (
-        db.session.query(
-            # return current month as a datetime so the data has the same shape as the ft_notification_status query
-            literal(fetch_day.replace(day=1), type_=DateTime).label("month"),
-            Notification.notification_type,
-            Notification.status.label("notification_status"),
-            func.count().label("count"),
-        )
-        .filter(
-            Notification.created_at >= get_midnight_in_utc(fetch_day),
-            Notification.created_at
-            < get_midnight_in_utc(fetch_day + timedelta(days=1)),
-            Notification.service_id == service_id,
-            Notification.key_type != KeyType.TEST,
-        )
-        .group_by(Notification.notification_type, Notification.status)
-        .all()
-    )
-
-
-def fetch_notification_status_for_service_for_today_and_7_previous_days(
-    service_id, by_template=False, limit_days=7
-):
-    start_date = midnight_n_days_ago(limit_days)
-    now = utc_now()
-    stats_for_7_days = db.session.query(
-        FactNotificationStatus.notification_type.cast(db.Text).label(
-            "notification_type"
-        ),
-        FactNotificationStatus.notification_status.cast(db.Text).label("status"),
-        *(
-            [FactNotificationStatus.template_id.label("template_id")]
-            if by_template
-            else []
-        ),
-        FactNotificationStatus.notification_count.label("count"),
+    return db.session.query(
+        func.date_trunc('month', FactNotificationStatus.local_date).label('month'),
+        FactNotificationStatus.notification_type,
+        FactNotificationStatus.notification_status,
+        func.sum(FactNotificationStatus.notification_count).label('count')
     ).filter(
         FactNotificationStatus.service_id == service_id,
         FactNotificationStatus.local_date >= start_date,
-        FactNotificationStatus.key_type != KeyType.TEST,
+        FactNotificationStatus.local_date < end_date,
+        FactNotificationStatus.key_type != KEY_TYPE_TEST
+    ).group_by(
+        func.date_trunc('month', FactNotificationStatus.local_date).label('month'),
+        FactNotificationStatus.notification_type,
+        FactNotificationStatus.notification_status
+    ).all()
+
+
+def fetch_notification_status_for_service_for_day(fetch_day, service_id):
+    return db.session.query(
+        # return current month as a datetime so the data has the same shape as the ft_notification_status query
+        literal(fetch_day.replace(day=1), type_=DateTime).label('month'),
+        Notification.notification_type,
+        Notification.status.label('notification_status'),
+        func.count().label('count')
+    ).filter(
+        Notification.created_at >= get_midnight_in_utc(fetch_day),
+        Notification.created_at < get_midnight_in_utc(fetch_day + timedelta(days=1)),
+        Notification.service_id == service_id,
+        Notification.key_type != KEY_TYPE_TEST
+    ).group_by(
+        Notification.notification_type,
+        Notification.status
+    ).all()
+
+
+def fetch_notification_status_for_service_for_today_and_7_previous_days(service_id, by_template=False, limit_days=7):
+    start_date = midnight_n_days_ago(limit_days)
+    now = datetime.utcnow()
+    stats_for_7_days = db.session.query(
+        FactNotificationStatus.notification_type.label('notification_type'),
+        FactNotificationStatus.notification_status.label('status'),
+        *([FactNotificationStatus.template_id.label('template_id')] if by_template else []),
+        FactNotificationStatus.notification_count.label('count')
+    ).filter(
+        FactNotificationStatus.service_id == service_id,
+        FactNotificationStatus.local_date >= start_date,
+        FactNotificationStatus.key_type != KEY_TYPE_TEST
     )
 
-    stats_for_today = (
-        db.session.query(
-            Notification.notification_type.cast(db.Text),
-            Notification.status.cast(db.Text),
-            *([Notification.template_id] if by_template else []),
-            func.count().label("count"),
-        )
-        .filter(
-            Notification.created_at >= get_midnight_in_utc(now),
-            Notification.service_id == service_id,
-            Notification.key_type != KeyType.TEST,
-        )
-        .group_by(
-            Notification.notification_type,
-            *([Notification.template_id] if by_template else []),
-            Notification.status,
-        )
+    stats_for_today = db.session.query(
+        Notification.notification_type.cast(db.Text),
+        Notification.status,
+        *([Notification.template_id] if by_template else []),
+        func.count().label('count')
+    ).filter(
+        Notification.created_at >= get_midnight_in_utc(now),
+        Notification.service_id == service_id,
+        Notification.key_type != KEY_TYPE_TEST
+    ).group_by(
+        Notification.notification_type,
+        *([Notification.template_id] if by_template else []),
+        Notification.status
     )
 
     all_stats_table = stats_for_7_days.union_all(stats_for_today).subquery()
 
     query = db.session.query(
-        *(
-            [
-                Template.name.label("template_name"),
-                False,  # TODO: this is related to is_precompiled_letter
-                all_stats_table.c.template_id,
-            ]
-            if by_template
-            else []
-        ),
+        *([
+            Template.name.label("template_name"),
+            False,  # TODO: this is related to is_precompiled_letter
+            all_stats_table.c.template_id
+        ] if by_template else []),
         all_stats_table.c.notification_type,
         all_stats_table.c.status,
-        func.cast(func.sum(all_stats_table.c.count), Integer).label("count"),
+        func.cast(func.sum(all_stats_table.c.count), Integer).label('count'),
     )
 
     if by_template:
@@ -194,169 +175,152 @@ def fetch_notification_status_for_service_for_today_and_7_previous_days(
 
 
 def fetch_notification_status_totals_for_all_services(start_date, end_date):
-    stats = (
-        db.session.query(
-            FactNotificationStatus.notification_type.cast(db.Text).label(
-                "notification_type"
-            ),
-            FactNotificationStatus.notification_status.cast(db.Text).label("status"),
-            FactNotificationStatus.key_type.cast(db.Text).label("key_type"),
-            func.sum(FactNotificationStatus.notification_count).label("count"),
-        )
-        .filter(
-            FactNotificationStatus.local_date >= start_date,
-            FactNotificationStatus.local_date <= end_date,
-        )
-        .group_by(
-            FactNotificationStatus.notification_type,
-            FactNotificationStatus.notification_status,
-            FactNotificationStatus.key_type,
-        )
+    stats = db.session.query(
+        FactNotificationStatus.notification_type.label('notification_type'),
+        FactNotificationStatus.notification_status.label('status'),
+        FactNotificationStatus.key_type.label('key_type'),
+        func.sum(FactNotificationStatus.notification_count).label('count')
+    ).filter(
+        FactNotificationStatus.local_date >= start_date,
+        FactNotificationStatus.local_date <= end_date
+    ).group_by(
+        FactNotificationStatus.notification_type,
+        FactNotificationStatus.notification_status,
+        FactNotificationStatus.key_type,
     )
-    today = get_midnight_in_utc(utc_now())
-    if start_date <= utc_now().date() <= end_date:
-        stats_for_today = (
-            db.session.query(
-                Notification.notification_type.cast(db.Text).label("notification_type"),
-                Notification.status.cast(db.Text),
-                Notification.key_type.cast(db.Text),
-                func.count().label("count"),
-            )
-            .filter(Notification.created_at >= today)
-            .group_by(
-                Notification.notification_type,
-                Notification.status,
-                Notification.key_type,
-            )
+    today = get_midnight_in_utc(datetime.utcnow())
+    if start_date <= datetime.utcnow().date() <= end_date:
+        stats_for_today = db.session.query(
+            Notification.notification_type.cast(db.Text).label('notification_type'),
+            Notification.status,
+            Notification.key_type,
+            func.count().label('count')
+        ).filter(
+            Notification.created_at >= today
+        ).group_by(
+            Notification.notification_type.cast(db.Text),
+            Notification.status,
+            Notification.key_type,
         )
         all_stats_table = stats.union_all(stats_for_today).subquery()
-        query = (
-            db.session.query(
-                all_stats_table.c.notification_type,
-                all_stats_table.c.status,
-                all_stats_table.c.key_type,
-                func.cast(func.sum(all_stats_table.c.count), Integer).label("count"),
-            )
-            .group_by(
-                all_stats_table.c.notification_type,
-                all_stats_table.c.status,
-                all_stats_table.c.key_type,
-            )
-            .order_by(all_stats_table.c.notification_type)
+        query = db.session.query(
+            all_stats_table.c.notification_type,
+            all_stats_table.c.status,
+            all_stats_table.c.key_type,
+            func.cast(func.sum(all_stats_table.c.count), Integer).label('count'),
+        ).group_by(
+            all_stats_table.c.notification_type,
+            all_stats_table.c.status,
+            all_stats_table.c.key_type,
+        ).order_by(
+            all_stats_table.c.notification_type
         )
     else:
-        query = stats.order_by(FactNotificationStatus.notification_type)
+        query = stats.order_by(
+            FactNotificationStatus.notification_type
+        )
     return query.all()
 
 
 def fetch_notification_statuses_for_job(job_id):
-    return (
-        db.session.query(
-            FactNotificationStatus.notification_status.label("status"),
-            func.sum(FactNotificationStatus.notification_count).label("count"),
-        )
-        .filter(
-            FactNotificationStatus.job_id == job_id,
-        )
-        .group_by(FactNotificationStatus.notification_status)
-        .all()
-    )
+    return db.session.query(
+        FactNotificationStatus.notification_status.label('status'),
+        func.sum(FactNotificationStatus.notification_count).label('count'),
+    ).filter(
+        FactNotificationStatus.job_id == job_id,
+    ).group_by(
+        FactNotificationStatus.notification_status
+    ).all()
 
 
-def fetch_stats_for_all_services_by_date_range(
-    start_date, end_date, include_from_test_key=True
-):
-    stats = (
-        db.session.query(
-            FactNotificationStatus.service_id.label("service_id"),
-            Service.name.label("name"),
-            Service.restricted.label("restricted"),
-            Service.active.label("active"),
-            Service.created_at.label("created_at"),
-            FactNotificationStatus.notification_type.cast(db.Text).label(
-                "notification_type"
-            ),
-            FactNotificationStatus.notification_status.cast(db.Text).label("status"),
-            func.sum(FactNotificationStatus.notification_count).label("count"),
-        )
-        .filter(
-            FactNotificationStatus.local_date >= start_date,
-            FactNotificationStatus.local_date <= end_date,
-            FactNotificationStatus.service_id == Service.id,
-        )
-        .group_by(
-            FactNotificationStatus.service_id.label("service_id"),
-            Service.name,
-            Service.restricted,
-            Service.active,
-            Service.created_at,
-            FactNotificationStatus.notification_type,
-            FactNotificationStatus.notification_status,
-        )
-        .order_by(
-            FactNotificationStatus.service_id, FactNotificationStatus.notification_type
-        )
+def fetch_stats_for_all_services_by_date_range(start_date, end_date, include_from_test_key=True):
+    stats = db.session.query(
+        FactNotificationStatus.service_id.label('service_id'),
+        Service.name.label('name'),
+        Service.restricted.label('restricted'),
+        Service.research_mode.label('research_mode'),
+        Service.active.label('active'),
+        Service.created_at.label('created_at'),
+        FactNotificationStatus.notification_type.label('notification_type'),
+        FactNotificationStatus.notification_status.label('status'),
+        func.sum(FactNotificationStatus.notification_count).label('count')
+    ).filter(
+        FactNotificationStatus.local_date >= start_date,
+        FactNotificationStatus.local_date <= end_date,
+        FactNotificationStatus.service_id == Service.id,
+    ).group_by(
+        FactNotificationStatus.service_id.label('service_id'),
+        Service.name,
+        Service.restricted,
+        Service.research_mode,
+        Service.active,
+        Service.created_at,
+        FactNotificationStatus.notification_type,
+        FactNotificationStatus.notification_status,
+    ).order_by(
+        FactNotificationStatus.service_id,
+        FactNotificationStatus.notification_type
     )
     if not include_from_test_key:
-        stats = stats.filter(FactNotificationStatus.key_type != KeyType.TEST)
+        stats = stats.filter(FactNotificationStatus.key_type != KEY_TYPE_TEST)
 
-    if start_date <= utc_now().date() <= end_date:
-        today = get_midnight_in_utc(utc_now())
-        subquery = (
-            db.session.query(
-                Notification.notification_type.label("notification_type"),
-                Notification.status.label("status"),
-                Notification.service_id.label("service_id"),
-                func.count(Notification.id).label("count"),
-            )
-            .filter(Notification.created_at >= today)
-            .group_by(
-                Notification.notification_type,
-                Notification.status,
-                Notification.service_id,
-            )
+    if start_date <= datetime.utcnow().date() <= end_date:
+        today = get_midnight_in_utc(datetime.utcnow())
+        subquery = db.session.query(
+            Notification.notification_type.cast(db.Text).label('notification_type'),
+            Notification.status.label('status'),
+            Notification.service_id.label('service_id'),
+            func.count(Notification.id).label('count')
+        ).filter(
+            Notification.created_at >= today
+        ).group_by(
+            Notification.notification_type,
+            Notification.status,
+            Notification.service_id
         )
         if not include_from_test_key:
-            subquery = subquery.filter(Notification.key_type != KeyType.TEST)
+            subquery = subquery.filter(Notification.key_type != KEY_TYPE_TEST)
         subquery = subquery.subquery()
 
         stats_for_today = db.session.query(
-            Service.id.label("service_id"),
-            Service.name.label("name"),
-            Service.restricted.label("restricted"),
-            Service.active.label("active"),
-            Service.created_at.label("created_at"),
-            subquery.c.notification_type.cast(db.Text).label("notification_type"),
-            subquery.c.status.cast(db.Text).label("status"),
-            subquery.c.count.label("count"),
-        ).outerjoin(subquery, subquery.c.service_id == Service.id)
+            Service.id.label('service_id'),
+            Service.name.label('name'),
+            Service.restricted.label('restricted'),
+            Service.research_mode.label('research_mode'),
+            Service.active.label('active'),
+            Service.created_at.label('created_at'),
+            subquery.c.notification_type.label('notification_type'),
+            subquery.c.status.label('status'),
+            subquery.c.count.label('count')
+        ).outerjoin(
+            subquery,
+            subquery.c.service_id == Service.id
+        )
 
         all_stats_table = stats.union_all(stats_for_today).subquery()
-        query = (
-            db.session.query(
-                all_stats_table.c.service_id,
-                all_stats_table.c.name,
-                all_stats_table.c.restricted,
-                all_stats_table.c.active,
-                all_stats_table.c.created_at,
-                all_stats_table.c.notification_type,
-                all_stats_table.c.status,
-                func.cast(func.sum(all_stats_table.c.count), Integer).label("count"),
-            )
-            .group_by(
-                all_stats_table.c.service_id,
-                all_stats_table.c.name,
-                all_stats_table.c.restricted,
-                all_stats_table.c.active,
-                all_stats_table.c.created_at,
-                all_stats_table.c.notification_type,
-                all_stats_table.c.status,
-            )
-            .order_by(
-                all_stats_table.c.name,
-                all_stats_table.c.notification_type,
-                all_stats_table.c.status,
-            )
+        query = db.session.query(
+            all_stats_table.c.service_id,
+            all_stats_table.c.name,
+            all_stats_table.c.restricted,
+            all_stats_table.c.research_mode,
+            all_stats_table.c.active,
+            all_stats_table.c.created_at,
+            all_stats_table.c.notification_type,
+            all_stats_table.c.status,
+            func.cast(func.sum(all_stats_table.c.count), Integer).label('count'),
+        ).group_by(
+            all_stats_table.c.service_id,
+            all_stats_table.c.name,
+            all_stats_table.c.restricted,
+            all_stats_table.c.research_mode,
+            all_stats_table.c.active,
+            all_stats_table.c.created_at,
+            all_stats_table.c.notification_type,
+            all_stats_table.c.status,
+        ).order_by(
+            all_stats_table.c.name,
+            all_stats_table.c.notification_type,
+            all_stats_table.c.status
         )
     else:
         query = stats
@@ -365,89 +329,77 @@ def fetch_stats_for_all_services_by_date_range(
 
 def fetch_monthly_template_usage_for_service(start_date, end_date, service_id):
     # services_dao.replaces dao_fetch_monthly_historical_usage_by_template_for_service
-    stats = (
-        db.session.query(
-            FactNotificationStatus.template_id.label("template_id"),
-            Template.name.label("name"),
-            Template.template_type.label("template_type"),
-            extract("month", FactNotificationStatus.local_date).label("month"),
-            extract("year", FactNotificationStatus.local_date).label("year"),
-            func.sum(FactNotificationStatus.notification_count).label("count"),
-        )
-        .join(Template, FactNotificationStatus.template_id == Template.id)
-        .filter(
-            FactNotificationStatus.service_id == service_id,
-            FactNotificationStatus.local_date >= start_date,
-            FactNotificationStatus.local_date <= end_date,
-            FactNotificationStatus.key_type != KeyType.TEST,
-            FactNotificationStatus.notification_status != NotificationStatus.CANCELLED,
-        )
-        .group_by(
-            FactNotificationStatus.template_id,
-            Template.name,
-            Template.template_type,
-            extract("month", FactNotificationStatus.local_date).label("month"),
-            extract("year", FactNotificationStatus.local_date).label("year"),
-        )
-        .order_by(
-            extract("year", FactNotificationStatus.local_date),
-            extract("month", FactNotificationStatus.local_date),
-            Template.name,
-        )
+    stats = db.session.query(
+        FactNotificationStatus.template_id.label('template_id'),
+        Template.name.label('name'),
+        Template.template_type.label('template_type'),
+        extract('month', FactNotificationStatus.local_date).label('month'),
+        extract('year', FactNotificationStatus.local_date).label('year'),
+        func.sum(FactNotificationStatus.notification_count).label('count')
+    ).join(
+        Template, FactNotificationStatus.template_id == Template.id
+    ).filter(
+        FactNotificationStatus.service_id == service_id,
+        FactNotificationStatus.local_date >= start_date,
+        FactNotificationStatus.local_date <= end_date,
+        FactNotificationStatus.key_type != KEY_TYPE_TEST,
+        FactNotificationStatus.notification_status != NOTIFICATION_CANCELLED,
+    ).group_by(
+        FactNotificationStatus.template_id,
+        Template.name,
+        Template.template_type,
+        extract('month', FactNotificationStatus.local_date).label('month'),
+        extract('year', FactNotificationStatus.local_date).label('year'),
+    ).order_by(
+        extract('year', FactNotificationStatus.local_date),
+        extract('month', FactNotificationStatus.local_date),
+        Template.name
     )
 
-    if start_date <= utc_now() <= end_date:
-        today = get_midnight_in_utc(utc_now())
+    if start_date <= datetime.utcnow() <= end_date:
+        today = get_midnight_in_utc(datetime.utcnow())
         month = get_month_from_utc_column(Notification.created_at)
 
-        stats_for_today = (
-            db.session.query(
-                Notification.template_id.label("template_id"),
-                Template.name.label("name"),
-                Template.template_type.label("template_type"),
-                extract("month", month).label("month"),
-                extract("year", month).label("year"),
-                func.count().label("count"),
-            )
-            .join(
-                Template,
-                Notification.template_id == Template.id,
-            )
-            .filter(
-                Notification.created_at >= today,
-                Notification.service_id == service_id,
-                Notification.key_type != KeyType.TEST,
-                Notification.status != NotificationStatus.CANCELLED,
-            )
-            .group_by(
-                Notification.template_id,
-                Template.hidden,
-                Template.name,
-                Template.template_type,
-                month,
-            )
+        stats_for_today = db.session.query(
+            Notification.template_id.label('template_id'),
+            Template.name.label('name'),
+            Template.template_type.label('template_type'),
+            extract('month', month).label('month'),
+            extract('year', month).label('year'),
+            func.count().label('count')
+        ).join(
+            Template, Notification.template_id == Template.id,
+        ).filter(
+            Notification.created_at >= today,
+            Notification.service_id == service_id,
+            Notification.key_type != KEY_TYPE_TEST,
+            Notification.status != NOTIFICATION_CANCELLED
+        ).group_by(
+            Notification.template_id,
+            Template.hidden,
+            Template.name,
+            Template.template_type,
+            month
         )
 
         all_stats_table = stats.union_all(stats_for_today).subquery()
-        query = (
-            db.session.query(
-                all_stats_table.c.template_id,
-                all_stats_table.c.name,
-                all_stats_table.c.template_type,
-                func.cast(all_stats_table.c.month, Integer).label("month"),
-                func.cast(all_stats_table.c.year, Integer).label("year"),
-                func.cast(func.sum(all_stats_table.c.count), Integer).label("count"),
-            )
-            .group_by(
-                all_stats_table.c.template_id,
-                all_stats_table.c.name,
-                all_stats_table.c.template_type,
-                all_stats_table.c.month,
-                all_stats_table.c.year,
-            )
-            .order_by(
-                all_stats_table.c.year, all_stats_table.c.month, all_stats_table.c.name
-            )
+        query = db.session.query(
+            all_stats_table.c.template_id,
+            all_stats_table.c.name,
+            all_stats_table.c.template_type,
+            func.cast(all_stats_table.c.month, Integer).label('month'),
+            func.cast(all_stats_table.c.year, Integer).label('year'),
+            func.cast(func.sum(all_stats_table.c.count), Integer).label('count'),
+        ).group_by(
+            all_stats_table.c.template_id,
+            all_stats_table.c.name,
+            all_stats_table.c.template_type,
+            all_stats_table.c.month,
+            all_stats_table.c.year,
+        ).order_by(
+            all_stats_table.c.year,
+            all_stats_table.c.month,
+            all_stats_table.c.name
         )
     else:
         query = stats
@@ -455,138 +407,92 @@ def fetch_monthly_template_usage_for_service(start_date, end_date, service_id):
 
 
 def get_total_notifications_for_date_range(start_date, end_date):
-    query = (
-        db.session.query(
-            FactNotificationStatus.local_date.label("local_date"),
-            func.sum(
-                case(
-                    (
-                        FactNotificationStatus.notification_type
-                        == NotificationType.EMAIL,
-                        FactNotificationStatus.notification_count,
-                    ),
-                    else_=0,
-                )
-            ).label("emails"),
-            func.sum(
-                case(
-                    (
-                        FactNotificationStatus.notification_type
-                        == NotificationType.SMS,
-                        FactNotificationStatus.notification_count,
-                    ),
-                    else_=0,
-                )
-            ).label("sms"),
-        )
-        .filter(
-            FactNotificationStatus.key_type != KeyType.TEST,
-        )
-        .group_by(FactNotificationStatus.local_date)
-        .order_by(FactNotificationStatus.local_date)
+    query = db.session.query(
+        FactNotificationStatus.local_date.cast(db.Text).label("local_date"),
+        func.sum(case(
+            [
+                (FactNotificationStatus.notification_type == 'email', FactNotificationStatus.notification_count)
+            ],
+            else_=0)).label('emails'),
+        func.sum(case(
+            [
+                (FactNotificationStatus.notification_type == 'sms', FactNotificationStatus.notification_count)
+            ],
+            else_=0)).label('sms'),
+    ).filter(
+        FactNotificationStatus.key_type != KEY_TYPE_TEST,
+    ).group_by(
+        FactNotificationStatus.local_date
+    ).order_by(
+        FactNotificationStatus.local_date
     )
     if start_date and end_date:
         query = query.filter(
             FactNotificationStatus.local_date >= start_date,
-            FactNotificationStatus.local_date <= end_date,
+            FactNotificationStatus.local_date <= end_date
         )
     return query.all()
 
 
 def fetch_monthly_notification_statuses_per_service(start_date, end_date):
-    return (
-        db.session.query(
-            func.date_trunc("month", FactNotificationStatus.local_date)
-            .cast(Date)
-            .label("date_created"),
-            Service.id.label("service_id"),
-            Service.name.label("service_name"),
-            FactNotificationStatus.notification_type,
-            func.sum(
-                case(
-                    (
-                        FactNotificationStatus.notification_status.in_(
-                            [NotificationStatus.SENDING, NotificationStatus.PENDING]
-                        ),
-                        FactNotificationStatus.notification_count,
-                    ),
-                    else_=0,
-                )
-            ).label("count_sending"),
-            func.sum(
-                case(
-                    (
-                        FactNotificationStatus.notification_status
-                        == NotificationStatus.DELIVERED,
-                        FactNotificationStatus.notification_count,
-                    ),
-                    else_=0,
-                )
-            ).label("count_delivered"),
-            func.sum(
-                case(
-                    (
-                        FactNotificationStatus.notification_status.in_(
-                            [
-                                NotificationStatus.TECHNICAL_FAILURE,
-                                NotificationStatus.FAILED,
-                            ]
-                        ),
-                        FactNotificationStatus.notification_count,
-                    ),
-                    else_=0,
-                )
-            ).label("count_technical_failure"),
-            func.sum(
-                case(
-                    (
-                        FactNotificationStatus.notification_status
-                        == NotificationStatus.TEMPORARY_FAILURE,
-                        FactNotificationStatus.notification_count,
-                    ),
-                    else_=0,
-                )
-            ).label("count_temporary_failure"),
-            func.sum(
-                case(
-                    (
-                        FactNotificationStatus.notification_status
-                        == NotificationStatus.PERMANENT_FAILURE,
-                        FactNotificationStatus.notification_count,
-                    ),
-                    else_=0,
-                )
-            ).label("count_permanent_failure"),
-            func.sum(
-                case(
-                    (
-                        FactNotificationStatus.notification_status
-                        == NotificationStatus.SENT,
-                        FactNotificationStatus.notification_count,
-                    ),
-                    else_=0,
-                )
-            ).label("count_sent"),
-        )
-        .join(Service, FactNotificationStatus.service_id == Service.id)
-        .filter(
-            FactNotificationStatus.notification_status != NotificationStatus.CREATED,
-            Service.active.is_(True),
-            FactNotificationStatus.key_type != KeyType.TEST,
-            Service.restricted.is_(False),
-            FactNotificationStatus.local_date >= start_date,
-            FactNotificationStatus.local_date <= end_date,
-        )
-        .group_by(
-            Service.id,
-            Service.name,
-            func.date_trunc("month", FactNotificationStatus.local_date).cast(Date),
-            FactNotificationStatus.notification_type,
-        )
-        .order_by(
-            func.date_trunc("month", FactNotificationStatus.local_date).cast(Date),
-            Service.id,
-            FactNotificationStatus.notification_type,
-        )
-        .all()
-    )
+    return db.session.query(
+        func.date_trunc('month', FactNotificationStatus.local_date).cast(Date).label('date_created'),
+        Service.id.label('service_id'),
+        Service.name.label('service_name'),
+        FactNotificationStatus.notification_type,
+        func.sum(case(
+            [
+                (FactNotificationStatus.notification_status.in_([NOTIFICATION_SENDING, NOTIFICATION_PENDING]),
+                 FactNotificationStatus.notification_count)
+            ],
+            else_=0)).label('count_sending'),
+        func.sum(case(
+            [
+                (FactNotificationStatus.notification_status == NOTIFICATION_DELIVERED,
+                 FactNotificationStatus.notification_count)
+            ],
+            else_=0)).label('count_delivered'),
+        func.sum(case(
+            [
+                (FactNotificationStatus.notification_status.in_([NOTIFICATION_TECHNICAL_FAILURE, NOTIFICATION_FAILED]),
+                 FactNotificationStatus.notification_count)
+            ],
+            else_=0)).label('count_technical_failure'),
+        func.sum(case(
+            [
+                (FactNotificationStatus.notification_status == NOTIFICATION_TEMPORARY_FAILURE,
+                 FactNotificationStatus.notification_count)
+            ],
+            else_=0)).label('count_temporary_failure'),
+        func.sum(case(
+            [
+                (FactNotificationStatus.notification_status == NOTIFICATION_PERMANENT_FAILURE,
+                 FactNotificationStatus.notification_count)
+            ],
+            else_=0)).label('count_permanent_failure'),
+        func.sum(case(
+            [
+                (FactNotificationStatus.notification_status == NOTIFICATION_SENT,
+                 FactNotificationStatus.notification_count)
+            ],
+            else_=0)).label('count_sent'),
+    ).join(
+        Service, FactNotificationStatus.service_id == Service.id
+    ).filter(
+        FactNotificationStatus.notification_status != NOTIFICATION_CREATED,
+        Service.active.is_(True),
+        FactNotificationStatus.key_type != KEY_TYPE_TEST,
+        Service.research_mode.is_(False),
+        Service.restricted.is_(False),
+        FactNotificationStatus.local_date >= start_date,
+        FactNotificationStatus.local_date <= end_date,
+    ).group_by(
+        Service.id,
+        Service.name,
+        func.date_trunc('month', FactNotificationStatus.local_date).cast(Date),
+        FactNotificationStatus.notification_type,
+    ).order_by(
+        func.date_trunc('month', FactNotificationStatus.local_date).cast(Date),
+        Service.id,
+        FactNotificationStatus.notification_type,
+    ).all()

app/dao/fact_processing_time_dao.py

@@ -1,62 +1,53 @@
+from datetime import datetime
+
 from sqlalchemy.dialects.postgresql import insert
 from sqlalchemy.sql.expression import case
 
 from app import db
 from app.dao.dao_utils import autocommit
 from app.models import FactProcessingTime
-from app.utils import utc_now
 
 
 @autocommit
 def insert_update_processing_time(processing_time):
-    """
+    '''
     This uses the Postgres upsert to avoid race conditions when two threads try and insert
     at the same row. The excluded object refers to values that we tried to insert but were
     rejected.
     http://docs.sqlalchemy.org/en/latest/dialects/postgresql.html#insert-on-conflict-upsert
-    """
+    '''
     table = FactProcessingTime.__table__
     stmt = insert(table).values(
         local_date=processing_time.local_date,
         messages_total=processing_time.messages_total,
-        messages_within_10_secs=processing_time.messages_within_10_secs,
+        messages_within_10_secs=processing_time.messages_within_10_secs
     )
     stmt = stmt.on_conflict_do_update(
         index_elements=[table.c.local_date],
         set_={
-            "messages_total": stmt.excluded.messages_total,
-            "messages_within_10_secs": stmt.excluded.messages_within_10_secs,
-            "updated_at": utc_now(),
-        },
+            'messages_total': stmt.excluded.messages_total,
+            'messages_within_10_secs': stmt.excluded.messages_within_10_secs,
+            'updated_at': datetime.utcnow()
+        }
     )
     db.session.connection().execute(stmt)
 
 
 def get_processing_time_percentage_for_date_range(start_date, end_date):
-    query = (
-        db.session.query(
-            FactProcessingTime.local_date.cast(db.Text).label("date"),
-            FactProcessingTime.messages_total,
-            FactProcessingTime.messages_within_10_secs,
-            case(
-                (
-                    FactProcessingTime.messages_total > 0,
-                    (
-                        (
-                            FactProcessingTime.messages_within_10_secs
-                            / FactProcessingTime.messages_total.cast(db.Float)
-                        )
-                        * 100
-                    ),
-                ),
-                (FactProcessingTime.messages_total == 0, 100.0),
-            ).label("percentage"),
-        )
-        .filter(
-            FactProcessingTime.local_date >= start_date,
-            FactProcessingTime.local_date <= end_date,
-        )
-        .order_by(FactProcessingTime.local_date)
-    )
+    query = db.session.query(
+        FactProcessingTime.local_date.cast(db.Text).label("date"),
+        FactProcessingTime.messages_total,
+        FactProcessingTime.messages_within_10_secs,
+        case([
+            (
+                FactProcessingTime.messages_total > 0,
+                ((FactProcessingTime.messages_within_10_secs / FactProcessingTime.messages_total.cast(db.Float)) * 100)
+            ),
+            (FactProcessingTime.messages_total == 0, 100.0)
+        ]).label("percentage")
+    ).filter(
+        FactProcessingTime.local_date >= start_date,
+        FactProcessingTime.local_date <= end_date
+    ).order_by(FactProcessingTime.local_date)
 
     return query.all()

app/dao/inbound_numbers_dao.py

@@ -8,9 +8,7 @@ def dao_get_inbound_numbers():
 
 
 def dao_get_available_inbound_numbers():
-    return InboundNumber.query.filter(
-        InboundNumber.active, InboundNumber.service_id.is_(None)
-    ).all()
+    return InboundNumber.query.filter(InboundNumber.active, InboundNumber.service_id.is_(None)).all()
 
 
 def dao_get_inbound_number_for_service(service_id):
@@ -29,9 +27,7 @@ def dao_set_inbound_number_to_service(service_id, inbound_number):
 
 @autocommit
 def dao_set_inbound_number_active_flag(service_id, active):
-    inbound_number = InboundNumber.query.filter(
-        InboundNumber.service_id == service_id
-    ).first()
+    inbound_number = InboundNumber.query.filter(InboundNumber.service_id == service_id).first()
     inbound_number.active = active
 
     db.session.add(inbound_number)
@@ -40,8 +36,12 @@ def dao_set_inbound_number_active_flag(service_id, active):
 @autocommit
 def dao_allocate_number_for_service(service_id, inbound_number_id):
     updated = InboundNumber.query.filter_by(
-        id=inbound_number_id, active=True, service_id=None
-    ).update({"service_id": service_id})
+        id=inbound_number_id,
+        active=True,
+        service_id=None
+    ).update(
+        {"service_id": service_id}
+    )
     if not updated:
         raise Exception("Inbound number: {} is not available".format(inbound_number_id))
     return InboundNumber.query.get(inbound_number_id)

app/dao/inbound_sms_dao.py

@@ -5,8 +5,13 @@ from sqlalchemy.orm import aliased
 
 from app import db
 from app.dao.dao_utils import autocommit
-from app.enums import NotificationType
-from app.models import InboundSms, InboundSmsHistory, ServiceDataRetention
+from app.models import (
+    SMS_TYPE,
+    InboundSms,
+    InboundSmsHistory,
+    Service,
+    ServiceDataRetention,
+)
 from app.utils import midnight_n_days_ago
 
 
@@ -15,10 +20,10 @@ def dao_create_inbound_sms(inbound_sms):
     db.session.add(inbound_sms)
 
 
-def dao_get_inbound_sms_for_service(
-    service_id, user_number=None, *, limit_days=None, limit=None
-):
-    q = InboundSms.query.filter(InboundSms.service_id == service_id).order_by(
+def dao_get_inbound_sms_for_service(service_id, user_number=None, *, limit_days=None, limit=None):
+    q = InboundSms.query.filter(
+        InboundSms.service_id == service_id
+    ).order_by(
         InboundSms.created_at.desc()
     )
     if limit_days is not None:
@@ -35,32 +40,31 @@ def dao_get_inbound_sms_for_service(
 
 
 def dao_get_paginated_inbound_sms_for_service_for_public_api(
-    service_id, older_than=None, page_size=None
+    service_id,
+    older_than=None,
+    page_size=None
 ):
     if page_size is None:
-        page_size = current_app.config["PAGE_SIZE"]
+        page_size = current_app.config['PAGE_SIZE']
 
     filters = [InboundSms.service_id == service_id]
 
     if older_than:
-        older_than_created_at = (
-            db.session.query(InboundSms.created_at)
-            .filter(InboundSms.id == older_than)
-            .as_scalar()
-        )
+        older_than_created_at = db.session.query(
+            InboundSms.created_at).filter(InboundSms.id == older_than).as_scalar()
         filters.append(InboundSms.created_at < older_than_created_at)
 
     query = InboundSms.query.filter(*filters)
 
-    return (
-        query.order_by(desc(InboundSms.created_at)).paginate(per_page=page_size).items
-    )
+    return query.order_by(desc(InboundSms.created_at)).paginate(
+        per_page=page_size
+    ).items
 
 
 def dao_count_inbound_sms_for_service(service_id, limit_days):
     return InboundSms.query.filter(
         InboundSms.service_id == service_id,
-        InboundSms.created_at >= midnight_n_days_ago(limit_days),
+        InboundSms.created_at >= midnight_n_days_ago(limit_days)
     ).count()
 
 
@@ -73,14 +77,14 @@ def _insert_inbound_sms_history(subquery, query_limit=10000):
         InboundSms.notify_number,
         InboundSms.provider_date,
         InboundSms.provider_reference,
-        InboundSms.provider,
+        InboundSms.provider
     ).filter(InboundSms.id.in_(subquery))
     inbound_sms_count = inbound_sms_query.count()
 
     while offset < inbound_sms_count:
         statement = insert(InboundSmsHistory).from_select(
             InboundSmsHistory.__table__.c,
-            inbound_sms_query.limit(query_limit).offset(offset),
+            inbound_sms_query.limit(query_limit).offset(offset)
         )
 
         statement = statement.on_conflict_do_nothing(
@@ -94,12 +98,14 @@ def _insert_inbound_sms_history(subquery, query_limit=10000):
 def _delete_inbound_sms(datetime_to_delete_from, query_filter):
     query_limit = 10000
 
-    subquery = (
-        db.session.query(InboundSms.id)
-        .filter(InboundSms.created_at < datetime_to_delete_from, *query_filter)
-        .limit(query_limit)
-        .subquery()
-    )
+    subquery = db.session.query(
+        InboundSms.id
+    ).filter(
+        InboundSms.created_at < datetime_to_delete_from,
+        *query_filter
+    ).limit(
+        query_limit
+    ).subquery()
 
     deleted = 0
     # set to nonzero just to enter the loop
@@ -107,9 +113,7 @@ def _delete_inbound_sms(datetime_to_delete_from, query_filter):
     while number_deleted > 0:
         _insert_inbound_sms_history(subquery, query_limit=query_limit)
 
-        number_deleted = InboundSms.query.filter(InboundSms.id.in_(subquery)).delete(
-            synchronize_session="fetch"
-        )
+        number_deleted = InboundSms.query.filter(InboundSms.id.in_(subquery)).delete(synchronize_session='fetch')
         deleted += number_deleted
 
     return deleted
@@ -117,52 +121,47 @@ def _delete_inbound_sms(datetime_to_delete_from, query_filter):
 
 @autocommit
 def delete_inbound_sms_older_than_retention():
-    current_app.logger.info(
-        "Deleting inbound sms for services with flexible data retention"
-    )
+    current_app.logger.info('Deleting inbound sms for services with flexible data retention')
 
-    flexible_data_retention = (
-        ServiceDataRetention.query.join(ServiceDataRetention.service)
-        .filter(ServiceDataRetention.notification_type == NotificationType.SMS)
-        .all()
-    )
+    flexible_data_retention = ServiceDataRetention.query.join(
+        ServiceDataRetention.service,
+        Service.inbound_number
+    ).filter(
+        ServiceDataRetention.notification_type == SMS_TYPE
+    ).all()
 
     deleted = 0
 
     for f in flexible_data_retention:
         n_days_ago = midnight_n_days_ago(f.days_of_retention)
 
-        current_app.logger.info(
-            "Deleting inbound sms for service id: {}".format(f.service_id)
-        )
-        deleted += _delete_inbound_sms(
-            n_days_ago, query_filter=[InboundSms.service_id == f.service_id]
-        )
+        current_app.logger.info("Deleting inbound sms for service id: {}".format(f.service_id))
+        deleted += _delete_inbound_sms(n_days_ago, query_filter=[InboundSms.service_id == f.service_id])
 
-    current_app.logger.info(
-        "Deleting inbound sms for services without flexible data retention"
-    )
+    current_app.logger.info('Deleting inbound sms for services without flexible data retention')
 
     seven_days_ago = midnight_n_days_ago(7)
 
-    deleted += _delete_inbound_sms(
-        seven_days_ago,
-        query_filter=[
-            InboundSms.service_id.notin_(x.service_id for x in flexible_data_retention),
-        ],
-    )
+    deleted += _delete_inbound_sms(seven_days_ago, query_filter=[
+        InboundSms.service_id.notin_(x.service_id for x in flexible_data_retention),
+    ])
 
-    current_app.logger.info("Deleted {} inbound sms".format(deleted))
+    current_app.logger.info('Deleted {} inbound sms'.format(deleted))
 
     return deleted
 
 
 def dao_get_inbound_sms_by_id(service_id, inbound_id):
-    return InboundSms.query.filter_by(id=inbound_id, service_id=service_id).one()
+    return InboundSms.query.filter_by(
+        id=inbound_id,
+        service_id=service_id
+    ).one()
 
 
 def dao_get_paginated_most_recent_inbound_sms_by_user_number_for_service(
-    service_id, page, limit_days
+    service_id,
+    page,
+    limit_days
 ):
     """
     This query starts from inbound_sms and joins on to itself to find the most recent row for each user_number.
@@ -183,22 +182,24 @@ def dao_get_paginated_most_recent_inbound_sms_by_user_number_for_service(
     LIMIT 50 OFFSET :page
     """
     t2 = aliased(InboundSms)
-    q = (
-        db.session.query(InboundSms)
-        .outerjoin(
-            t2,
-            and_(
-                InboundSms.user_number == t2.user_number,
-                InboundSms.service_id == t2.service_id,
-                InboundSms.created_at < t2.created_at,
-            ),
+    q = db.session.query(
+        InboundSms
+    ).outerjoin(
+        t2,
+        and_(
+            InboundSms.user_number == t2.user_number,
+            InboundSms.service_id == t2.service_id,
+            InboundSms.created_at < t2.created_at,
         )
-        .filter(
-            t2.id == None,  # noqa
-            InboundSms.service_id == service_id,
-            InboundSms.created_at >= midnight_n_days_ago(limit_days),
-        )
-        .order_by(InboundSms.created_at.desc())
+    ).filter(
+        t2.id == None,  # noqa
+        InboundSms.service_id == service_id,
+        InboundSms.created_at >= midnight_n_days_ago(limit_days)
+    ).order_by(
+        InboundSms.created_at.desc()
     )
 
-    return q.paginate(page=page, per_page=current_app.config["PAGE_SIZE"])
+    return q.paginate(
+        page=page,
+        per_page=current_app.config['PAGE_SIZE']
+    )

app/dao/invited_org_user_dao.py

@@ -1,8 +1,7 @@
-from datetime import timedelta
+from datetime import datetime, timedelta
 
 from app import db
 from app.models import InvitedOrganizationUser
-from app.utils import utc_now
 
 
 def save_invited_org_user(invited_org_user):
@@ -11,9 +10,7 @@ def save_invited_org_user(invited_org_user):
 
 
 def get_invited_org_user(organization_id, invited_org_user_id):
-    return InvitedOrganizationUser.query.filter_by(
-        organization_id=organization_id, id=invited_org_user_id
-    ).one()
+    return InvitedOrganizationUser.query.filter_by(organization_id=organization_id, id=invited_org_user_id).one()
 
 
 def get_invited_org_user_by_id(invited_org_user_id):
@@ -21,16 +18,12 @@ def get_invited_org_user_by_id(invited_org_user_id):
 
 
 def get_invited_org_users_for_organization(organization_id):
-    return InvitedOrganizationUser.query.filter_by(
-        organization_id=organization_id
-    ).all()
+    return InvitedOrganizationUser.query.filter_by(organization_id=organization_id).all()
 
 
 def delete_org_invitations_created_more_than_two_days_ago():
-    deleted = (
-        db.session.query(InvitedOrganizationUser)
-        .filter(InvitedOrganizationUser.created_at <= utc_now() - timedelta(days=2))
-        .delete()
-    )
+    deleted = db.session.query(InvitedOrganizationUser).filter(
+        InvitedOrganizationUser.created_at <= datetime.utcnow() - timedelta(days=2)
+    ).delete()
     db.session.commit()
     return deleted

app/dao/invited_user_dao.py

@@ -1,9 +1,7 @@
-from datetime import timedelta
+from datetime import datetime, timedelta
 
 from app import db
-from app.enums import InvitedUserStatus
 from app.models import InvitedUser
-from app.utils import utc_now
 
 
 def save_invited_user(invited_user):
@@ -12,40 +10,20 @@ def save_invited_user(invited_user):
 
 
 def get_invited_user_by_service_and_id(service_id, invited_user_id):
-    return InvitedUser.query.filter(
-        InvitedUser.service_id == service_id,
-        InvitedUser.id == invited_user_id,
-    ).one()
-
-
-def get_expired_invite_by_service_and_id(service_id, invited_user_id):
-    return InvitedUser.query.filter(
-        InvitedUser.service_id == service_id,
-        InvitedUser.id == invited_user_id,
-        InvitedUser.status == InvitedUserStatus.EXPIRED,
-    ).one()
+    return InvitedUser.query.filter_by(service_id=service_id, id=invited_user_id).one()
 
 
 def get_invited_user_by_id(invited_user_id):
-    return InvitedUser.query.filter(InvitedUser.id == invited_user_id).one()
-
-
-def get_expired_invited_users_for_service(service_id):
-    return InvitedUser.query.filter(InvitedUser.service_id == service_id).all()
+    return InvitedUser.query.filter_by(id=invited_user_id).one()
 
 
 def get_invited_users_for_service(service_id):
-    return InvitedUser.query.filter(InvitedUser.service_id == service_id).all()
+    return InvitedUser.query.filter_by(service_id=service_id).all()
 
 
-def expire_invitations_created_more_than_two_days_ago():
-    expired = (
-        db.session.query(InvitedUser)
-        .filter(
-            InvitedUser.created_at <= utc_now() - timedelta(days=2),
-            InvitedUser.status.in_((InvitedUserStatus.PENDING,)),
-        )
-        .update({InvitedUser.status: InvitedUserStatus.EXPIRED})
-    )
+def delete_invitations_created_more_than_two_days_ago():
+    deleted = db.session.query(InvitedUser).filter(
+        InvitedUser.created_at <= datetime.utcnow() - timedelta(days=2)
+    ).delete()
     db.session.commit()
-    return expired
+    return deleted

app/dao/jobs_dao.py

@@ -1,43 +1,41 @@
 import uuid
-from datetime import timedelta
+from datetime import datetime, timedelta
 
 from flask import current_app
 from sqlalchemy import and_, asc, desc, func
 
 from app import db
-from app.enums import JobStatus
 from app.models import (
+    JOB_STATUS_FINISHED,
+    JOB_STATUS_PENDING,
+    JOB_STATUS_SCHEDULED,
     FactNotificationStatus,
     Job,
     Notification,
     ServiceDataRetention,
     Template,
 )
-from app.utils import midnight_n_days_ago, utc_now
+from app.utils import midnight_n_days_ago
 
 
 def dao_get_notification_outcomes_for_job(service_id, job_id):
-    notification_statuses = (
-        db.session.query(
-            func.count(Notification.status).label("count"), Notification.status
-        )
-        .filter(Notification.service_id == service_id, Notification.job_id == job_id)
-        .group_by(Notification.status)
-        .all()
-    )
+    notification_statuses = db.session.query(
+        func.count(Notification.status).label('count'), Notification.status
+    ).filter(
+        Notification.service_id == service_id,
+        Notification.job_id == job_id
+    ).group_by(
+        Notification.status
+    ).all()
 
     if not notification_statuses:
-        notification_statuses = (
-            db.session.query(
-                FactNotificationStatus.notification_count.label("count"),
-                FactNotificationStatus.notification_status.label("status"),
-            )
-            .filter(
-                FactNotificationStatus.service_id == service_id,
-                FactNotificationStatus.job_id == job_id,
-            )
-            .all()
-        )
+        notification_statuses = db.session.query(
+            FactNotificationStatus.notification_count.label('count'),
+            FactNotificationStatus.notification_status.label('status')
+        ).filter(
+            FactNotificationStatus.service_id == service_id,
+            FactNotificationStatus.job_id == job_id
+        ).all()
     return notification_statuses
 
 
@@ -59,34 +57,31 @@ def dao_get_jobs_by_service_id(
 ):
     query_filter = [
         Job.service_id == service_id,
-        Job.original_file_name != current_app.config["TEST_MESSAGE_FILENAME"],
-        Job.original_file_name != current_app.config["ONE_OFF_MESSAGE_FILENAME"],
+        Job.original_file_name != current_app.config['TEST_MESSAGE_FILENAME'],
+        Job.original_file_name != current_app.config['ONE_OFF_MESSAGE_FILENAME'],
     ]
     if limit_days is not None:
         query_filter.append(Job.created_at >= midnight_n_days_ago(limit_days))
-    if statuses is not None and statuses != [""]:
-        query_filter.append(Job.job_status.in_(statuses))
-    return (
-        Job.query.filter(*query_filter)
-        .order_by(Job.processing_started.desc(), Job.created_at.desc())
+    if statuses is not None and statuses != ['']:
+        query_filter.append(
+            Job.job_status.in_(statuses)
+        )
+    return Job.query \
+        .filter(*query_filter) \
+        .order_by(Job.processing_started.desc(), Job.created_at.desc()) \
         .paginate(page=page, per_page=page_size)
-    )
 
 
 def dao_get_scheduled_job_stats(
     service_id,
 ):
-    return (
-        db.session.query(
-            func.count(Job.id),
-            func.min(Job.scheduled_for),
-        )
-        .filter(
-            Job.service_id == service_id,
-            Job.job_status == JobStatus.SCHEDULED,
-        )
-        .one()
-    )
+    return db.session.query(
+        func.count(Job.id),
+        func.min(Job.scheduled_for),
+    ).filter(
+        Job.service_id == service_id,
+        Job.job_status == JOB_STATUS_SCHEDULED,
+    ).one()
 
 
 def dao_get_job_by_id(job_id):
@@ -107,18 +102,17 @@ def dao_set_scheduled_jobs_to_pending():
     the transaction so that if the task is run more than once concurrently, one task will block the other select
     from completing until it commits.
     """
-    jobs = (
-        Job.query.filter(
-            Job.job_status == JobStatus.SCHEDULED,
-            Job.scheduled_for < utc_now(),
-        )
-        .order_by(asc(Job.scheduled_for))
-        .with_for_update()
+    jobs = Job.query \
+        .filter(
+            Job.job_status == JOB_STATUS_SCHEDULED,
+            Job.scheduled_for < datetime.utcnow()
+        ) \
+        .order_by(asc(Job.scheduled_for)) \
+        .with_for_update() \
         .all()
-    )
 
     for job in jobs:
-        job.job_status = JobStatus.PENDING
+        job.job_status = JOB_STATUS_PENDING
 
     db.session.add_all(jobs)
     db.session.commit()
@@ -127,12 +121,14 @@ def dao_set_scheduled_jobs_to_pending():
 
 
 def dao_get_future_scheduled_job_by_id_and_service_id(job_id, service_id):
-    return Job.query.filter(
-        Job.service_id == service_id,
-        Job.id == job_id,
-        Job.job_status == JobStatus.SCHEDULED,
-        Job.scheduled_for > utc_now(),
-    ).one()
+    return Job.query \
+        .filter(
+            Job.service_id == service_id,
+            Job.id == job_id,
+            Job.job_status == JOB_STATUS_SCHEDULED,
+            Job.scheduled_for > datetime.utcnow()
+        ) \
+        .one()
 
 
 def dao_create_job(job):
@@ -152,40 +148,28 @@ def dao_get_jobs_older_than_data_retention(notification_types):
         ServiceDataRetention.notification_type.in_(notification_types)
     ).all()
     jobs = []
-    today = utc_now().date()
+    today = datetime.utcnow().date()
     for f in flexible_data_retention:
         end_date = today - timedelta(days=f.days_of_retention)
 
-        jobs.extend(
-            Job.query.join(Template)
-            .filter(
-                func.coalesce(Job.scheduled_for, Job.created_at) < end_date,
-                Job.archived == False,  # noqa
-                Template.template_type == f.notification_type,
-                Job.service_id == f.service_id,
-            )
-            .order_by(desc(Job.created_at))
-            .all()
-        )
+        jobs.extend(Job.query.join(Template).filter(
+            func.coalesce(Job.scheduled_for, Job.created_at) < end_date,
+            Job.archived == False,  # noqa
+            Template.template_type == f.notification_type,
+            Job.service_id == f.service_id
+        ).order_by(desc(Job.created_at)).all())
 
     end_date = today - timedelta(days=7)
     for notification_type in notification_types:
         services_with_data_retention = [
-            x.service_id
-            for x in flexible_data_retention
-            if x.notification_type == notification_type
+            x.service_id for x in flexible_data_retention if x.notification_type == notification_type
         ]
-        jobs.extend(
-            Job.query.join(Template)
-            .filter(
-                func.coalesce(Job.scheduled_for, Job.created_at) < end_date,
-                Job.archived == False,  # noqa
-                Template.template_type == notification_type,
-                Job.service_id.notin_(services_with_data_retention),
-            )
-            .order_by(desc(Job.created_at))
-            .all()
-        )
+        jobs.extend(Job.query.join(Template).filter(
+            func.coalesce(Job.scheduled_for, Job.created_at) < end_date,
+            Job.archived == False,  # noqa
+            Template.template_type == notification_type,
+            Job.service_id.notin_(services_with_data_retention)
+        ).order_by(desc(Job.created_at)).all())
 
     return jobs
 
@@ -193,18 +177,20 @@ def dao_get_jobs_older_than_data_retention(notification_types):
 def find_jobs_with_missing_rows():
     # Jobs can be a maximum of 100,000 rows. It typically takes 10 minutes to create all those notifications.
     # Using 20 minutes as a condition seems reasonable.
-    ten_minutes_ago = utc_now() - timedelta(minutes=20)
-    yesterday = utc_now() - timedelta(days=1)
-    jobs_with_rows_missing = (
-        db.session.query(Job)
-        .filter(
-            Job.job_status == JobStatus.FINISHED,
-            Job.processing_finished < ten_minutes_ago,
-            Job.processing_finished > yesterday,
-            Job.id == Notification.job_id,
-        )
-        .group_by(Job)
-        .having(func.count(Notification.id) != Job.notification_count)
+    ten_minutes_ago = datetime.utcnow() - timedelta(minutes=20)
+    yesterday = datetime.utcnow() - timedelta(days=1)
+    jobs_with_rows_missing = db.session.query(
+        Job
+    ).filter(
+        Job.job_status == JOB_STATUS_FINISHED,
+        Job.processing_finished < ten_minutes_ago,
+        Job.processing_finished > yesterday,
+        Job.id == Notification.job_id,
+
+    ).group_by(
+        Job
+    ).having(
+        func.count(Notification.id) != Job.notification_count
     )
 
     return jobs_with_rows_missing.all()
@@ -212,20 +198,15 @@ def find_jobs_with_missing_rows():
 
 def find_missing_row_for_job(job_id, job_size):
     expected_row_numbers = db.session.query(
-        func.generate_series(0, job_size - 1).label("row")
+        func.generate_series(0, job_size - 1).label('row')
     ).subquery()
 
-    query = (
-        db.session.query(
-            Notification.job_row_number, expected_row_numbers.c.row.label("missing_row")
-        )
-        .outerjoin(
-            Notification,
-            and_(
-                expected_row_numbers.c.row == Notification.job_row_number,
-                Notification.job_id == job_id,
-            ),
-        )
-        .filter(Notification.job_row_number == None)  # noqa
+    query = db.session.query(
+        Notification.job_row_number,
+        expected_row_numbers.c.row.label('missing_row')
+    ).outerjoin(
+        Notification, and_(expected_row_numbers.c.row == Notification.job_row_number, Notification.job_id == job_id)
+    ).filter(
+        Notification.job_row_number == None  # noqa
     )
     return query.all()

app/dao/notifications_dao.py

@@ -1,7 +1,15 @@
-from datetime import timedelta
+from datetime import datetime, timedelta
 
 from flask import current_app
-from sqlalchemy import asc, desc, or_, select, text, union
+from notifications_utils.international_billing_rates import (
+    INTERNATIONAL_BILLING_RATES,
+)
+from notifications_utils.recipients import (
+    InvalidEmailError,
+    try_validate_and_format_phone_number,
+    validate_and_format_email_address,
+)
+from sqlalchemy import asc, desc, func, or_, union
 from sqlalchemy.orm import joinedload
 from sqlalchemy.orm.exc import NoResultFound
 from sqlalchemy.sql import functions
@@ -10,44 +18,46 @@ from werkzeug.datastructures import MultiDict
 
 from app import create_uuid, db
 from app.dao.dao_utils import autocommit
-from app.enums import KeyType, NotificationStatus, NotificationType
-from app.models import FactNotificationStatus, Notification, NotificationHistory
+from app.models import (
+    EMAIL_TYPE,
+    KEY_TYPE_TEST,
+    NOTIFICATION_CREATED,
+    NOTIFICATION_PENDING,
+    NOTIFICATION_PENDING_VIRUS_CHECK,
+    NOTIFICATION_PERMANENT_FAILURE,
+    NOTIFICATION_SENDING,
+    NOTIFICATION_SENT,
+    NOTIFICATION_TEMPORARY_FAILURE,
+    SMS_TYPE,
+    FactNotificationStatus,
+    Notification,
+    NotificationHistory,
+)
 from app.utils import (
     escape_special_characters,
     get_midnight_in_utc,
     midnight_n_days_ago,
-    utc_now,
-)
-from notifications_utils.international_billing_rates import INTERNATIONAL_BILLING_RATES
-from notifications_utils.recipients import (
-    InvalidEmailError,
-    try_validate_and_format_phone_number,
-    validate_and_format_email_address,
 )
 
 
 def dao_get_last_date_template_was_used(template_id, service_id):
-    last_date_from_notifications = (
-        db.session.query(functions.max(Notification.created_at))
-        .filter(
-            Notification.service_id == service_id,
-            Notification.template_id == template_id,
-            Notification.key_type != KeyType.TEST,
-        )
-        .scalar()
-    )
+    last_date_from_notifications = db.session.query(
+        functions.max(Notification.created_at)
+    ).filter(
+        Notification.service_id == service_id,
+        Notification.template_id == template_id,
+        Notification.key_type != KEY_TYPE_TEST
+    ).scalar()
 
     if last_date_from_notifications:
         return last_date_from_notifications
 
-    last_date = (
-        db.session.query(functions.max(FactNotificationStatus.local_date))
-        .filter(
-            FactNotificationStatus.template_id == template_id,
-            FactNotificationStatus.key_type != KeyType.TEST,
-        )
-        .scalar()
-    )
+    last_date = db.session.query(
+        functions.max(FactNotificationStatus.local_date)
+    ).filter(
+        FactNotificationStatus.template_id == template_id,
+        FactNotificationStatus.key_type != KEY_TYPE_TEST
+    ).scalar()
 
     return last_date
 
@@ -58,144 +68,105 @@ def dao_create_notification(notification):
         # need to populate defaulted fields before we create the notification history object
         notification.id = create_uuid()
     if not notification.status:
-        notification.status = NotificationStatus.CREATED
+        notification.status = NOTIFICATION_CREATED
 
-    # notify-api-749 do not write to db
-    # if we have a verify_code we know this is the authentication notification at login time
-    # and not csv (containing PII) provided by the user, so allow verify_code to continue to exist
-    if "verify_code" in str(notification.personalisation):
-        pass
-    else:
-        notification.personalisation = ""
-
-    # notify-api-742 remove phone numbers from db
-    notification.to = "1"
-    notification.normalised_to = "1"
     db.session.add(notification)
 
 
 def country_records_delivery(phone_prefix):
-    dlr = INTERNATIONAL_BILLING_RATES[phone_prefix]["attributes"]["dlr"]
-    return dlr and dlr.lower() == "yes"
+    dlr = INTERNATIONAL_BILLING_RATES[phone_prefix]['attributes']['dlr']
+    return dlr and dlr.lower() == 'yes'
 
 
 def _decide_permanent_temporary_failure(current_status, status):
     # If we go from pending to delivered we need to set failure type as temporary-failure
-    if (
-        current_status == NotificationStatus.PENDING
-        and status == NotificationStatus.PERMANENT_FAILURE
-    ):
-        status = NotificationStatus.TEMPORARY_FAILURE
+    if current_status == NOTIFICATION_PENDING and status == NOTIFICATION_PERMANENT_FAILURE:
+        status = NOTIFICATION_TEMPORARY_FAILURE
     return status
 
 
-def _update_notification_status(
-    notification, status, provider_response=None, carrier=None
-):
-    status = _decide_permanent_temporary_failure(
-        current_status=notification.status, status=status
-    )
+def _update_notification_status(notification, status, provider_response=None):
+    status = _decide_permanent_temporary_failure(current_status=notification.status, status=status)
     notification.status = status
-    notification.sent_at = utc_now()
     if provider_response:
         notification.provider_response = provider_response
-    if carrier:
-        notification.carrier = carrier
     dao_update_notification(notification)
     return notification
 
 
 @autocommit
-def update_notification_status_by_id(
-    notification_id, status, sent_by=None, provider_response=None, carrier=None
-):
-    notification = (
-        Notification.query.with_for_update()
-        .filter(Notification.id == notification_id)
-        .first()
-    )
+def update_notification_status_by_id(notification_id, status, sent_by=None, provider_response=None):
+    notification = Notification.query.with_for_update().filter(Notification.id == notification_id).first()
 
     if not notification:
-        current_app.logger.info(
-            "notification not found for id {} (update to status {})".format(
-                notification_id, status
-            )
-        )
+        current_app.logger.info('notification not found for id {} (update to status {})'.format(
+            notification_id,
+            status
+        ))
         return None
 
     if notification.status not in {
-        NotificationStatus.CREATED,
-        NotificationStatus.SENDING,
-        NotificationStatus.PENDING,
-        NotificationStatus.SENT,
-        NotificationStatus.PENDING_VIRUS_CHECK,
+        NOTIFICATION_CREATED,
+        NOTIFICATION_SENDING,
+        NOTIFICATION_PENDING,
+        NOTIFICATION_SENT,
+        NOTIFICATION_PENDING_VIRUS_CHECK
     }:
         _duplicate_update_warning(notification, status)
         return None
 
     if (
-        notification.notification_type == NotificationType.SMS
+        notification.notification_type == SMS_TYPE
         and notification.international
         and not country_records_delivery(notification.phone_prefix)
     ):
         return None
     if provider_response:
         notification.provider_response = provider_response
-    if carrier:
-        notification.carrier = carrier
     if not notification.sent_by and sent_by:
         notification.sent_by = sent_by
     return _update_notification_status(
         notification=notification,
-        status=status,
-        provider_response=notification.provider_response,
-        carrier=notification.carrier,
+        status=status
     )
 
 
 @autocommit
 def update_notification_status_by_reference(reference, status):
     # this is used to update emails
-    notification = Notification.query.filter(
-        Notification.reference == reference
-    ).first()
+    notification = Notification.query.filter(Notification.reference == reference).first()
 
     if not notification:
-        current_app.logger.error(
-            "notification not found for reference {} (update to {})".format(
-                reference, status
-            )
-        )
+        current_app.logger.error('notification not found for reference {} (update to {})'.format(reference, status))
         return None
 
     if notification.status not in {
-        NotificationStatus.SENDING,
-        NotificationStatus.PENDING,
+        NOTIFICATION_SENDING,
+        NOTIFICATION_PENDING
     }:
         _duplicate_update_warning(notification, status)
         return None
 
-    return _update_notification_status(notification=notification, status=status)
+    return _update_notification_status(
+        notification=notification,
+        status=status
+    )
 
 
 @autocommit
 def dao_update_notification(notification):
-    notification.updated_at = utc_now()
-    # notify-api-742 remove phone numbers from db
-    notification.to = "1"
-    notification.normalised_to = "1"
+    notification.updated_at = datetime.utcnow()
     db.session.add(notification)
 
 
-def get_notifications_for_job(
-    service_id, job_id, filter_dict=None, page=1, page_size=None
-):
+def get_notifications_for_job(service_id, job_id, filter_dict=None, page=1, page_size=None):
     if page_size is None:
-        page_size = current_app.config["PAGE_SIZE"]
+        page_size = current_app.config['PAGE_SIZE']
     query = Notification.query.filter_by(service_id=service_id, job_id=job_id)
     query = _filter_query(query, filter_dict)
     return query.order_by(asc(Notification.job_row_number)).paginate(
-        page=page, per_page=page_size
+        page=page,
+        per_page=page_size
     )
 
 
@@ -203,28 +174,12 @@ def dao_get_notification_count_for_job_id(*, job_id):
     return Notification.query.filter_by(job_id=job_id).count()
 
 
-def dao_get_notification_count_for_service(*, service_id):
-    notification_count = Notification.query.filter_by(service_id=service_id).count()
-    return notification_count
-
-
-def dao_get_failed_notification_count():
-    failed_count = Notification.query.filter_by(
-        status=NotificationStatus.FAILED
-    ).count()
-    return failed_count
-
-
 def get_notification_with_personalisation(service_id, notification_id, key_type):
-    filter_dict = {"service_id": service_id, "id": notification_id}
+    filter_dict = {'service_id': service_id, 'id': notification_id}
     if key_type:
-        filter_dict["key_type"] = key_type
+        filter_dict['key_type'] = key_type
 
-    return (
-        Notification.query.filter_by(**filter_dict)
-        .options(joinedload(Notification.template))
-        .one()
-    )
+    return Notification.query.filter_by(**filter_dict).options(joinedload('template')).one()
 
 
 def get_notification_by_id(notification_id, service_id=None, _raise=False):
@@ -239,23 +194,23 @@ def get_notification_by_id(notification_id, service_id=None, _raise=False):
 
 
 def get_notifications_for_service(
-    service_id,
-    filter_dict=None,
-    page=1,
-    page_size=None,
-    count_pages=True,
-    limit_days=None,
-    key_type=None,
-    personalisation=False,
-    include_jobs=False,
-    include_from_test_key=False,
-    older_than=None,
-    client_reference=None,
-    include_one_off=True,
-    error_out=True,
+        service_id,
+        filter_dict=None,
+        page=1,
+        page_size=None,
+        count_pages=True,
+        limit_days=None,
+        key_type=None,
+        personalisation=False,
+        include_jobs=False,
+        include_from_test_key=False,
+        older_than=None,
+        client_reference=None,
+        include_one_off=True,
+        error_out=True
 ):
     if page_size is None:
-        page_size = current_app.config["PAGE_SIZE"]
+        page_size = current_app.config['PAGE_SIZE']
 
     filters = [Notification.service_id == service_id]
 
@@ -263,11 +218,8 @@ def get_notifications_for_service(
         filters.append(Notification.created_at >= midnight_n_days_ago(limit_days))
 
     if older_than is not None:
-        older_than_created_at = (
-            db.session.query(Notification.created_at)
-            .filter(Notification.id == older_than)
-            .as_scalar()
-        )
+        older_than_created_at = db.session.query(
+            Notification.created_at).filter(Notification.id == older_than).as_scalar()
         filters.append(Notification.created_at < older_than_created_at)
 
     if not include_jobs:
@@ -279,7 +231,7 @@ def get_notifications_for_service(
     if key_type is not None:
         filters.append(Notification.key_type == key_type)
     elif not include_from_test_key:
-        filters.append(Notification.key_type != KeyType.TEST)
+        filters.append(Notification.key_type != KEY_TYPE_TEST)
 
     if client_reference is not None:
         filters.append(Notification.client_reference == client_reference)
@@ -287,7 +239,9 @@ def get_notifications_for_service(
     query = Notification.query.filter(*filters)
     query = _filter_query(query, filter_dict)
     if personalisation:
-        query = query.options(joinedload(Notification.template))
+        query = query.options(
+            joinedload('template')
+        )
 
     return query.order_by(desc(Notification.created_at)).paginate(
         page=page,
@@ -304,35 +258,41 @@ def _filter_query(query, filter_dict=None):
     multidict = MultiDict(filter_dict)
 
     # filter by status
-    statuses = multidict.getlist("status")
+    statuses = multidict.getlist('status')
 
     if statuses:
         query = query.filter(Notification.status.in_(statuses))
 
     # filter by template
-    template_types = multidict.getlist("template_type")
+    template_types = multidict.getlist('template_type')
     if template_types:
         query = query.filter(Notification.notification_type.in_(template_types))
 
     return query
 
 
-def sanitize_successful_notification_by_id(notification_id, carrier, provider_response):
+@autocommit
+def sanitize_successful_notification_by_id(
+    notification_id
+):
+    # TODO what to do for international?
+    # phone_prefix = '1'
+    # Notification.query.filter(
+    #     Notification.id.in_([notification_id]),
+    # ).update(
+    #     {'to': phone_prefix, 'normalised_to': phone_prefix, 'status': 'delivered'}
+    # )
+    # db.session.commit()
+
     update_query = """
-    update notifications set provider_response=:response, carrier=:carrier,
-    notification_status='delivered', sent_at=:sent_at, "to"='1', normalised_to='1'
+    update notifications set notification_status='delivered', "to"='1', normalised_to='1'
     where id=:notification_id
     """
-
     input_params = {
-        "notification_id": notification_id,
-        "carrier": carrier,
-        "response": provider_response,
-        "sent_at": utc_now(),
+        "notification_id": notification_id
     }
 
-    db.session.execute(text(update_query), input_params)
-    db.session.commit()
+    db.session.execute(update_query, input_params)
 
 
 @autocommit
@@ -384,24 +344,25 @@ def insert_notification_history_delete_notifications(
         "service_id": service_id,
         "notification_type": notification_type,
         "timestamp_to_delete_backwards_from": timestamp_to_delete_backwards_from,
-        "qry_limit": qry_limit,
+        "qry_limit": qry_limit
     }
 
-    db.session.execute(text(select_into_temp_table), input_params)
+    db.session.execute(select_into_temp_table, input_params)
 
-    result = db.session.execute(
-        text("select count(*) from NOTIFICATION_ARCHIVE")
-    ).fetchone()[0]
+    result = db.session.execute("select count(*) from NOTIFICATION_ARCHIVE").fetchone()[0]
 
-    db.session.execute(text(insert_query))
+    db.session.execute(insert_query)
 
-    db.session.execute(text(delete_query))
+    db.session.execute(delete_query)
 
     return result
 
 
 def move_notifications_to_notification_history(
-    notification_type, service_id, timestamp_to_delete_backwards_from, qry_limit=50000
+    notification_type,
+    service_id,
+    timestamp_to_delete_backwards_from,
+    qry_limit=50000
 ):
     deleted = 0
     delete_count_per_call = 1
@@ -410,7 +371,7 @@ def move_notifications_to_notification_history(
             notification_type=notification_type,
             service_id=service_id,
             timestamp_to_delete_backwards_from=timestamp_to_delete_backwards_from,
-            qry_limit=qry_limit,
+            qry_limit=qry_limit
         )
         deleted += delete_count_per_call
 
@@ -419,7 +380,7 @@ def move_notifications_to_notification_history(
         Notification.notification_type == notification_type,
         Notification.service_id == service_id,
         Notification.created_at < timestamp_to_delete_backwards_from,
-        Notification.key_type == KeyType.TEST,
+        Notification.key_type == KEY_TYPE_TEST
     ).delete(synchronize_session=False)
     db.session.commit()
 
@@ -428,9 +389,9 @@ def move_notifications_to_notification_history(
 
 @autocommit
 def dao_delete_notifications_by_id(notification_id):
-    db.session.query(Notification).filter(Notification.id == notification_id).delete(
-        synchronize_session="fetch"
-    )
+    db.session.query(Notification).filter(
+        Notification.id == notification_id
+    ).delete(synchronize_session='fetch')
 
 
 def dao_timeout_notifications(cutoff_time, limit=100000):
@@ -438,26 +399,21 @@ def dao_timeout_notifications(cutoff_time, limit=100000):
     Set email and SMS notifications (only) to "temporary-failure" status
     if they're still sending from before the specified cutoff_time.
     """
-    updated_at = utc_now()
-    current_statuses = [NotificationStatus.SENDING, NotificationStatus.PENDING]
-    new_status = NotificationStatus.TEMPORARY_FAILURE
+    updated_at = datetime.utcnow()
+    current_statuses = [NOTIFICATION_SENDING, NOTIFICATION_PENDING]
+    new_status = NOTIFICATION_TEMPORARY_FAILURE
 
-    notifications = (
-        Notification.query.filter(
-            Notification.created_at < cutoff_time,
-            Notification.status.in_(current_statuses),
-            Notification.notification_type.in_(
-                [NotificationType.SMS, NotificationType.EMAIL]
-            ),
-        )
-        .limit(limit)
-        .all()
-    )
+    notifications = Notification.query.filter(
+        Notification.created_at < cutoff_time,
+        Notification.status.in_(current_statuses),
+        Notification.notification_type.in_([SMS_TYPE, EMAIL_TYPE])
+    ).limit(limit).all()
 
     Notification.query.filter(
         Notification.id.in_([n.id for n in notifications]),
     ).update(
-        {"status": new_status, "updated_at": updated_at}, synchronize_session=False
+        {'status': new_status, 'updated_at': updated_at},
+        synchronize_session=False
     )
 
     db.session.commit()
@@ -468,13 +424,19 @@ def dao_timeout_notifications(cutoff_time, limit=100000):
 def dao_update_notifications_by_reference(references, update_dict):
     updated_count = Notification.query.filter(
         Notification.reference.in_(references)
-    ).update(update_dict, synchronize_session=False)
+    ).update(
+        update_dict,
+        synchronize_session=False
+    )
 
     updated_history_count = 0
     if updated_count != len(references):
         updated_history_count = NotificationHistory.query.filter(
             NotificationHistory.reference.in_(references)
-        ).update(update_dict, synchronize_session=False)
+        ).update(
+            update_dict,
+            synchronize_session=False
+        )
 
     return updated_count, updated_history_count
 
@@ -488,15 +450,16 @@ def dao_get_notifications_by_recipient_or_reference(
     page_size=None,
     error_out=True,
 ):
-    if notification_type == NotificationType.SMS:
+
+    if notification_type == SMS_TYPE:
         normalised = try_validate_and_format_phone_number(search_term)
 
-        for character in {"(", ")", " ", "-"}:
-            normalised = normalised.replace(character, "")
+        for character in {'(', ')', ' ', '-'}:
+            normalised = normalised.replace(character, '')
 
-        normalised = normalised.lstrip("+0")
+        normalised = normalised.lstrip('+0')
 
-    elif notification_type == NotificationType.EMAIL:
+    elif notification_type == EMAIL_TYPE:
         try:
             normalised = validate_and_format_email_address(search_term)
         except InvalidEmailError:
@@ -507,11 +470,11 @@ def dao_get_notifications_by_recipient_or_reference(
         # happen if a user doesn’t have permission to see the dashboard)
         # because email addresses and phone numbers will never be stored
         # with spaces either.
-        normalised = "".join(search_term.split()).lower()
+        normalised = ''.join(search_term.split()).lower()
 
     else:
         raise TypeError(
-            f"Notification type must be {NotificationType.EMAIL}, {NotificationType.SMS}, or None"
+            f'Notification type must be {EMAIL_TYPE}, {SMS_TYPE}, or None'
         )
 
     normalised = escape_special_characters(normalised)
@@ -523,7 +486,7 @@ def dao_get_notifications_by_recipient_or_reference(
             Notification.normalised_to.like("%{}%".format(normalised)),
             Notification.client_reference.ilike("%{}%".format(search_term)),
         ),
-        Notification.key_type != KeyType.TEST,
+        Notification.key_type != KEY_TYPE_TEST,
     ]
 
     if statuses:
@@ -531,17 +494,17 @@ def dao_get_notifications_by_recipient_or_reference(
     if notification_type:
         filters.append(Notification.notification_type == notification_type)
 
-    results = (
-        db.session.query(Notification)
-        .filter(*filters)
-        .order_by(desc(Notification.created_at))
+    results = db.session.query(Notification)\
+        .filter(*filters)\
+        .order_by(desc(Notification.created_at))\
         .paginate(page=page, per_page=page_size, count=False, error_out=error_out)
-    )
     return results
 
 
 def dao_get_notification_by_reference(reference):
-    return Notification.query.filter(Notification.reference == reference).one()
+    return Notification.query.filter(
+        Notification.reference == reference
+    ).one()
 
 
 def dao_get_notification_history_by_reference(reference):
@@ -550,9 +513,7 @@ def dao_get_notification_history_by_reference(reference):
         # Otherwise we could just search for the NotificationHistory object
         return Notification.query.filter(Notification.reference == reference).one()
     except NoResultFound:
-        return NotificationHistory.query.filter(
-            NotificationHistory.reference == reference
-        ).one()
+        return NotificationHistory.query.filter(NotificationHistory.reference == reference).one()
 
 
 def dao_get_notifications_processing_time_stats(start_date, end_date):
@@ -570,42 +531,44 @@ def dao_get_notifications_processing_time_stats(start_date, end_date):
     api_key_id IS NOT NULL AND
     key_type != 'test';
     """
-    under_10_secs = Notification.sent_at - Notification.created_at <= timedelta(
-        seconds=10
-    )
-    sum_column = functions.coalesce(functions.sum(case((under_10_secs, 1), else_=0)), 0)
+    under_10_secs = Notification.sent_at - Notification.created_at <= timedelta(seconds=10)
+    sum_column = functions.coalesce(functions.sum(
+        case(
+            [
+                (under_10_secs, 1)
+            ],
+            else_=0
+        )
+    ), 0)
 
-    stmt = select(
-        functions.count(Notification.id).label("messages_total"),
-        sum_column.label("messages_within_10_secs"),
-    ).where(
+    return db.session.query(
+        func.count(Notification.id).label('messages_total'),
+        sum_column.label('messages_within_10_secs')
+    ).filter(
         Notification.created_at >= start_date,
         Notification.created_at < end_date,
         Notification.api_key_id.isnot(None),
-        Notification.key_type != KeyType.TEST,
-    )
-
-    result = db.session.execute(stmt)
-    return result.one()
+        Notification.key_type != KEY_TYPE_TEST,
+    ).one()
 
 
 def dao_get_last_notification_added_for_job_id(job_id):
-    last_notification_added = (
-        Notification.query.filter(Notification.job_id == job_id)
-        .order_by(Notification.job_row_number.desc())
-        .first()
-    )
+    last_notification_added = Notification.query.filter(
+        Notification.job_id == job_id
+    ).order_by(
+        Notification.job_row_number.desc()
+    ).first()
 
     return last_notification_added
 
 
 def notifications_not_yet_sent(should_be_sending_after_seconds, notification_type):
-    older_than_date = utc_now() - timedelta(seconds=should_be_sending_after_seconds)
+    older_than_date = datetime.utcnow() - timedelta(seconds=should_be_sending_after_seconds)
 
     notifications = Notification.query.filter(
         Notification.created_at <= older_than_date,
         Notification.notification_type == notification_type,
-        Notification.status == NotificationStatus.CREATED,
+        Notification.status == NOTIFICATION_CREATED
     ).all()
     return notifications
 
@@ -613,18 +576,18 @@ def notifications_not_yet_sent(should_be_sending_after_seconds, notification_typ
 def _duplicate_update_warning(notification, status):
     current_app.logger.info(
         (
-            "Duplicate callback received for service {service_id}. "
-            "Notification ID {id} with type {type} sent by {sent_by}. "
-            "New status was {new_status}, current status is {old_status}. "
-            "This happened {time_diff} after being first set."
+            'Duplicate callback received for service {service_id}. '
+            'Notification ID {id} with type {type} sent by {sent_by}. '
+            'New status was {new_status}, current status is {old_status}. '
+            'This happened {time_diff} after being first set.'
         ).format(
             id=notification.id,
             old_status=notification.status,
             new_status=status,
-            time_diff=utc_now() - (notification.updated_at or notification.created_at),
+            time_diff=datetime.utcnow() - (notification.updated_at or notification.created_at),
             type=notification.notification_type,
             sent_by=notification.sent_by,
-            service_id=notification.service_id,
+            service_id=notification.service_id
         )
     )
 
@@ -632,12 +595,12 @@ def _duplicate_update_warning(notification, status):
 def get_service_ids_with_notifications_before(notification_type, timestamp):
     return {
         row.service_id
-        for row in db.session.query(Notification.service_id)
-        .filter(
+        for row in db.session.query(
+            Notification.service_id
+        ).filter(
             Notification.notification_type == notification_type,
-            Notification.created_at < timestamp,
-        )
-        .distinct()
+            Notification.created_at < timestamp
+        ).distinct()
     }
 
 
@@ -646,7 +609,7 @@ def get_service_ids_with_notifications_on_date(notification_type, date):
     end_date = get_midnight_in_utc(date + timedelta(days=1))
 
     notification_table_query = db.session.query(
-        Notification.service_id.label("service_id")
+        Notification.service_id.label('service_id')
     ).filter(
         Notification.notification_type == notification_type,
         # using >= + < is much more efficient than date(created_at)
@@ -657,15 +620,14 @@ def get_service_ids_with_notifications_on_date(notification_type, date):
     # Looking at this table is more efficient for historical notifications,
     # provided the task to populate it has run before they were archived.
     ft_status_table_query = db.session.query(
-        FactNotificationStatus.service_id.label("service_id")
+        FactNotificationStatus.service_id.label('service_id')
     ).filter(
         FactNotificationStatus.notification_type == notification_type,
         FactNotificationStatus.local_date == date,
     )
 
     return {
-        row.service_id
-        for row in db.session.query(
-            union(notification_table_query, ft_status_table_query).subquery()
-        ).distinct()
+        row.service_id for row in db.session.query(union(
+            notification_table_query, ft_status_table_query
+        ).subquery()).distinct()
     }

app/dao/organization_dao.py

@@ -12,26 +12,23 @@ def dao_get_organizations():
 
 
 def dao_count_organizations_with_live_services():
-    return (
-        db.session.query(Organization.id)
-        .join(Organization.services)
-        .filter(
-            Service.active.is_(True),
-            Service.restricted.is_(False),
-            Service.count_as_live.is_(True),
-        )
-        .distinct()
-        .count()
-    )
+    return db.session.query(Organization.id).join(Organization.services).filter(
+        Service.active.is_(True),
+        Service.restricted.is_(False),
+        Service.count_as_live.is_(True),
+    ).distinct().count()
 
 
 def dao_get_organization_services(organization_id):
-    return Organization.query.filter_by(id=organization_id).one().services
+    return Organization.query.filter_by(
+        id=organization_id
+    ).one().services
 
 
 def dao_get_organization_live_services(organization_id):
     return Service.query.filter_by(
-        organization_id=organization_id, restricted=False
+        organization_id=organization_id,
+        restricted=False
     ).all()
 
 
@@ -40,21 +37,22 @@ def dao_get_organization_by_id(organization_id):
 
 
 def dao_get_organization_by_email_address(email_address):
-    email_address = email_address.lower().replace(".gsi.gov.uk", ".gov.uk")
+
+    email_address = email_address.lower().replace('.gsi.gov.uk', '.gov.uk')
 
     for domain in Domain.query.order_by(func.char_length(Domain.domain).desc()).all():
-        if email_address.endswith(
-            "@{}".format(domain.domain)
-        ) or email_address.endswith(".{}".format(domain.domain)):
+
+        if (
+            email_address.endswith("@{}".format(domain.domain)) or
+            email_address.endswith(".{}".format(domain.domain))
+        ):
             return Organization.query.filter_by(id=domain.organization_id).one()
 
     return None
 
 
 def dao_get_organization_by_service_id(service_id):
-    return (
-        Organization.query.join(Organization.services).filter_by(id=service_id).first()
-    )
+    return Organization.query.join(Organization.services).filter_by(id=service_id).first()
 
 
 @autocommit
@@ -64,26 +62,24 @@ def dao_create_organization(organization):
 
 @autocommit
 def dao_update_organization(organization_id, **kwargs):
-    domains = kwargs.pop("domains", None)
-    num_updated = Organization.query.filter_by(id=organization_id).update(kwargs)
+    domains = kwargs.pop('domains', None)
+    num_updated = Organization.query.filter_by(id=organization_id).update(
+        kwargs
+    )
 
     if isinstance(domains, list):
         Domain.query.filter_by(organization_id=organization_id).delete()
-        db.session.bulk_save_objects(
-            [
-                Domain(domain=domain.lower(), organization_id=organization_id)
-                for domain in domains
-            ]
-        )
+        db.session.bulk_save_objects([
+            Domain(domain=domain.lower(), organization_id=organization_id)
+            for domain in domains
+        ])
 
     organization = Organization.query.get(organization_id)
-    if "organization_type" in kwargs:
-        _update_organization_services(
-            organization, "organization_type", only_where_none=False
-        )
+    if 'organization_type' in kwargs:
+        _update_organization_services(organization, 'organization_type', only_where_none=False)
 
-    if "email_branding_id" in kwargs:
-        _update_organization_services(organization, "email_branding")
+    if 'email_branding_id' in kwargs:
+        _update_organization_services(organization, 'email_branding')
 
     return num_updated
 
@@ -101,7 +97,9 @@ def _update_organization_services(organization, attribute, only_where_none=True)
 @autocommit
 @version_class(Service)
 def dao_add_service_to_organization(service, organization_id):
-    organization = Organization.query.filter_by(id=organization_id).one()
+    organization = Organization.query.filter_by(
+        id=organization_id
+    ).one()
 
     service.organization_id = organization_id
     service.organization_type = organization.organization_type
@@ -110,13 +108,14 @@ def dao_add_service_to_organization(service, organization_id):
 
 
 def dao_get_users_for_organization(organization_id):
-    return (
-        db.session.query(User)
-        .join(User.organizations)
-        .filter(Organization.id == organization_id, User.state == "active")
-        .order_by(User.created_at)
-        .all()
-    )
+    return db.session.query(
+        User
+    ).join(
+        User.organizations
+    ).filter(
+        Organization.id == organization_id,
+        User.state == 'active'
+    ).order_by(User.created_at).all()
 
 
 @autocommit

app/dao/permissions_dao.py

@@ -1,15 +1,34 @@
 from app import db
 from app.dao import DAOClass
-from app.enums import PermissionType
-from app.models import Permission
+from app.models import (
+    MANAGE_API_KEYS,
+    MANAGE_SETTINGS,
+    MANAGE_TEMPLATES,
+    MANAGE_USERS,
+    SEND_EMAILS,
+    SEND_TEXTS,
+    VIEW_ACTIVITY,
+    Permission,
+)
+
+# Default permissions for a service
+default_service_permissions = [
+    MANAGE_USERS,
+    MANAGE_TEMPLATES,
+    MANAGE_SETTINGS,
+    SEND_TEXTS,
+    SEND_EMAILS,
+    MANAGE_API_KEYS,
+    VIEW_ACTIVITY]
 
 
 class PermissionDAO(DAOClass):
+
     class Meta:
         model = Permission
 
     def add_default_service_permissions_for_user(self, user, service):
-        for name in PermissionType.defaults():
+        for name in default_service_permissions:
             permission = Permission(permission=name, user=user, service=service)
             self.create_instance(permission, _commit=False)
 
@@ -21,14 +40,10 @@ class PermissionDAO(DAOClass):
         query = self.Meta.model.query.filter_by(user=user)
         query.delete()
 
-    def set_user_service_permission(
-        self, user, service, permissions, _commit=False, replace=False
-    ):
+    def set_user_service_permission(self, user, service, permissions, _commit=False, replace=False):
         try:
             if replace:
-                query = self.Meta.model.query.filter(
-                    self.Meta.model.user == user, self.Meta.model.service == service
-                )
+                query = self.Meta.model.query.filter_by(user=user, service=service)
                 query.delete()
             for p in permissions:
                 p.user = user
@@ -43,20 +58,12 @@ class PermissionDAO(DAOClass):
                 db.session.commit()
 
     def get_permissions_by_user_id(self, user_id):
-        return (
-            self.Meta.model.query.filter_by(user_id=user_id)
-            .join(Permission.service)
-            .filter_by(active=True)
-            .all()
-        )
+        return self.Meta.model.query.filter_by(user_id=user_id)\
+                                    .join(Permission.service).filter_by(active=True).all()
 
     def get_permissions_by_user_id_and_service_id(self, user_id, service_id):
-        return (
-            self.Meta.model.query.filter_by(user_id=user_id)
-            .join(Permission.service)
-            .filter_by(active=True, id=service_id)
-            .all()
-        )
+        return self.Meta.model.query.filter_by(user_id=user_id)\
+                                    .join(Permission.service).filter_by(active=True, id=service_id).all()
 
 
 permission_dao = PermissionDAO()

app/dao/provider_details_dao.py

@@ -1,13 +1,17 @@
-from datetime import datetime
+from datetime import datetime, timedelta
 
 from flask import current_app
 from sqlalchemy import asc, desc, func
 
 from app import db
 from app.dao.dao_utils import autocommit
-from app.enums import NotificationType
-from app.models import FactBilling, ProviderDetails, ProviderDetailsHistory, User
-from app.utils import utc_now
+from app.models import (
+    SMS_TYPE,
+    FactBilling,
+    ProviderDetails,
+    ProviderDetailsHistory,
+    User,
+)
 
 
 def get_provider_details_by_id(provider_details_id):
@@ -19,28 +23,29 @@ def get_provider_details_by_identifier(identifier):
 
 
 def get_alternative_sms_provider(identifier):
-    if identifier == "sns":
+    if identifier == 'sns':
         raise Exception("No alternative SMS providers currently available")
-    raise ValueError("Unrecognised sms provider {}".format(identifier))
+    raise ValueError('Unrecognised sms provider {}'.format(identifier))
 
 
 def dao_get_provider_versions(provider_id):
-    return (
-        ProviderDetailsHistory.query.filter_by(id=provider_id)
-        .order_by(desc(ProviderDetailsHistory.version))
-        .limit(100)  # limit results instead of adding pagination
-        .all()
-    )
+    return ProviderDetailsHistory.query.filter_by(
+        id=provider_id
+    ).order_by(
+        desc(ProviderDetailsHistory.version)
+    ).limit(
+        100  # limit results instead of adding pagination
+    ).all()
 
 
 def _adjust_provider_priority(provider, new_priority):
     current_app.logger.info(
-        f"Adjusting provider priority - {provider.identifier} going from {provider.priority} to {new_priority}"
+        f'Adjusting provider priority - {provider.identifier} going from {provider.priority} to {new_priority}'
     )
     provider.priority = new_priority
 
     # Automatic update so set as notify user
-    provider.created_by_id = current_app.config["NOTIFY_USER_ID"]
+    provider.created_by_id = current_app.config['NOTIFY_USER_ID']
 
     # update without commit so that both rows can be changed without ending the transaction
     # and releasing the for_update lock
@@ -56,41 +61,78 @@ def _get_sms_providers_for_update(time_threshold):
     release the transaction in that case
     """
     # get current priority of both providers
-    q = (
-        ProviderDetails.query.filter(
-            ProviderDetails.notification_type == NotificationType.SMS,
-            ProviderDetails.active,
-        )
-        .with_for_update()
-        .all()
-    )
+    q = ProviderDetails.query.filter(
+        ProviderDetails.notification_type == 'sms',
+        ProviderDetails.active
+    ).with_for_update().all()
 
     # if something updated recently, don't update again. If the updated_at is null, treat it as min time
-    if any(
-        (provider.updated_at or datetime.min) > utc_now() - time_threshold
-        for provider in q
-    ):
-        current_app.logger.info(
-            f"Not adjusting providers, providers updated less than {time_threshold} ago."
-        )
+    if any((provider.updated_at or datetime.min) > datetime.utcnow() - time_threshold for provider in q):
+        current_app.logger.info(f"Not adjusting providers, providers updated less than {time_threshold} ago.")
         return []
 
     return q
 
 
-def get_provider_details_by_notification_type(
-    notification_type, supports_international=False
-):
+@autocommit
+def dao_reduce_sms_provider_priority(identifier, *, time_threshold):
+    """
+    Will reduce a chosen sms provider's priority, and increase the other provider's priority by 10 points each.
+    If either provider has been updated in the last `time_threshold`, then it won't take any action.
+    """
+    amount_to_reduce_by = 10
+    providers_list = _get_sms_providers_for_update(time_threshold)
+
+    if len(providers_list) < 2:
+        current_app.logger.info("Not adjusting providers, number of active providers is less than 2.")
+        return
+
+    providers = {provider.identifier: provider for provider in providers_list}
+    other_identifier = get_alternative_sms_provider(identifier)
+
+    reduced_provider = providers[identifier]
+    increased_provider = providers[other_identifier]
+
+    # always keep values between 0 and 100
+    reduced_provider_priority = max(0, reduced_provider.priority - amount_to_reduce_by)
+    increased_provider_priority = min(100, increased_provider.priority + amount_to_reduce_by)
+
+    _adjust_provider_priority(reduced_provider, reduced_provider_priority)
+    _adjust_provider_priority(increased_provider, increased_provider_priority)
+
+
+@autocommit
+def dao_adjust_provider_priority_back_to_resting_points():
+    """
+    Provided that neither SMS provider has been modified in the last hour, move both providers by 10 percentage points
+    each towards their defined resting points (set in SMS_PROVIDER_RESTING_POINTS in config.py).
+    """
+    amount_to_reduce_by = 10
+    time_threshold = timedelta(hours=1)
+
+    providers = _get_sms_providers_for_update(time_threshold)
+
+    for provider in providers:
+        target = current_app.config['SMS_PROVIDER_RESTING_POINTS'][provider.identifier]
+        current = provider.priority
+
+        if current != target:
+            if current > target:
+                new_priority = max(target, provider.priority - amount_to_reduce_by)
+            else:
+                new_priority = min(target, provider.priority + amount_to_reduce_by)
+
+            _adjust_provider_priority(provider, new_priority)
+
+
+def get_provider_details_by_notification_type(notification_type, supports_international=False):
+
     filters = [ProviderDetails.notification_type == notification_type]
 
     if supports_international:
         filters.append(ProviderDetails.supports_international == supports_international)
 
-    return (
-        ProviderDetails.query.filter(*filters)
-        .order_by(asc(ProviderDetails.priority))
-        .all()
-    )
+    return ProviderDetails.query.filter(*filters).order_by(asc(ProviderDetails.priority)).all()
 
 
 @autocommit
@@ -103,7 +145,7 @@ def _update_provider_details_without_commit(provider_details):
     Doesn't commit, for when you need to control the database transaction manually
     """
     provider_details.version += 1
-    provider_details.updated_at = utc_now()
+    provider_details.updated_at = datetime.utcnow()
     history = ProviderDetailsHistory.from_original(provider_details)
     db.session.add(provider_details)
     db.session.add(history)
@@ -112,46 +154,37 @@ def _update_provider_details_without_commit(provider_details):
 def dao_get_provider_stats():
     # this query does not include the current day since the task to populate ft_billing runs overnight
 
-    current_datetime = utc_now()
+    current_datetime = datetime.utcnow()
     first_day_of_the_month = current_datetime.date().replace(day=1)
 
-    subquery = (
-        db.session.query(
-            FactBilling.provider,
-            func.sum(FactBilling.billable_units * FactBilling.rate_multiplier).label(
-                "current_month_billable_sms"
-            ),
-        )
-        .filter(
-            FactBilling.notification_type == NotificationType.SMS,
-            FactBilling.local_date >= first_day_of_the_month,
-        )
-        .group_by(FactBilling.provider)
-        .subquery()
-    )
+    subquery = db.session.query(
+        FactBilling.provider,
+        func.sum(FactBilling.billable_units * FactBilling.rate_multiplier).label('current_month_billable_sms')
+    ).filter(
+        FactBilling.notification_type == SMS_TYPE,
+        FactBilling.local_date >= first_day_of_the_month
+    ).group_by(
+        FactBilling.provider
+    ).subquery()
 
-    result = (
-        db.session.query(
-            ProviderDetails.id,
-            ProviderDetails.display_name,
-            ProviderDetails.identifier,
-            ProviderDetails.priority,
-            ProviderDetails.notification_type,
-            ProviderDetails.active,
-            ProviderDetails.updated_at,
-            ProviderDetails.supports_international,
-            User.name.label("created_by_name"),
-            func.coalesce(subquery.c.current_month_billable_sms, 0).label(
-                "current_month_billable_sms"
-            ),
-        )
-        .outerjoin(subquery, ProviderDetails.identifier == subquery.c.provider)
-        .outerjoin(User, ProviderDetails.created_by_id == User.id)
-        .order_by(
-            ProviderDetails.notification_type,
-            ProviderDetails.priority,
-        )
-        .all()
-    )
+    result = db.session.query(
+        ProviderDetails.id,
+        ProviderDetails.display_name,
+        ProviderDetails.identifier,
+        ProviderDetails.priority,
+        ProviderDetails.notification_type,
+        ProviderDetails.active,
+        ProviderDetails.updated_at,
+        ProviderDetails.supports_international,
+        User.name.label('created_by_name'),
+        func.coalesce(subquery.c.current_month_billable_sms, 0).label('current_month_billable_sms')
+    ).outerjoin(
+        subquery, ProviderDetails.identifier == subquery.c.provider
+    ).outerjoin(
+        User, ProviderDetails.created_by_id == User.id
+    ).order_by(
+        ProviderDetails.notification_type,
+        ProviderDetails.priority,
+    ).all()
 
     return result

app/dao/service_callback_api_dao.py

@@ -1,50 +1,50 @@
+from datetime import datetime
+
 from app import create_uuid, db
 from app.dao.dao_utils import autocommit, version_class
-from app.enums import CallbackType
-from app.models import ServiceCallbackApi
-from app.utils import utc_now
+from app.models import (
+    COMPLAINT_CALLBACK_TYPE,
+    DELIVERY_STATUS_CALLBACK_TYPE,
+    ServiceCallbackApi,
+)
 
 
 @autocommit
 @version_class(ServiceCallbackApi)
 def save_service_callback_api(service_callback_api):
     service_callback_api.id = create_uuid()
-    service_callback_api.created_at = utc_now()
+    service_callback_api.created_at = datetime.utcnow()
     db.session.add(service_callback_api)
 
 
 @autocommit
 @version_class(ServiceCallbackApi)
-def reset_service_callback_api(
-    service_callback_api, updated_by_id, url=None, bearer_token=None
-):
+def reset_service_callback_api(service_callback_api, updated_by_id, url=None, bearer_token=None):
     if url:
         service_callback_api.url = url
     if bearer_token:
         service_callback_api.bearer_token = bearer_token
     service_callback_api.updated_by_id = updated_by_id
-    service_callback_api.updated_at = utc_now()
+    service_callback_api.updated_at = datetime.utcnow()
 
     db.session.add(service_callback_api)
 
 
 def get_service_callback_api(service_callback_api_id, service_id):
-    return ServiceCallbackApi.query.filter_by(
-        id=service_callback_api_id, service_id=service_id
-    ).first()
+    return ServiceCallbackApi.query.filter_by(id=service_callback_api_id, service_id=service_id).first()
 
 
 def get_service_delivery_status_callback_api_for_service(service_id):
     return ServiceCallbackApi.query.filter_by(
         service_id=service_id,
-        callback_type=CallbackType.DELIVERY_STATUS,
+        callback_type=DELIVERY_STATUS_CALLBACK_TYPE
     ).first()
 
 
 def get_service_complaint_callback_api_for_service(service_id):
     return ServiceCallbackApi.query.filter_by(
         service_id=service_id,
-        callback_type=CallbackType.COMPLAINT,
+        callback_type=COMPLAINT_CALLBACK_TYPE
     ).first()
 
 

app/dao/service_data_retention_dao.py

@@ -1,61 +1,56 @@
+from datetime import datetime
+
 from app import db
 from app.dao.dao_utils import autocommit
 from app.models import ServiceDataRetention
-from app.utils import utc_now
 
 
 def fetch_service_data_retention_by_id(service_id, data_retention_id):
-    data_retention = ServiceDataRetention.query.filter_by(
-        service_id=service_id, id=data_retention_id
-    ).first()
+    data_retention = ServiceDataRetention.query.filter_by(service_id=service_id, id=data_retention_id).first()
     return data_retention
 
 
 def fetch_service_data_retention(service_id):
-    data_retention_list = (
-        ServiceDataRetention.query.filter_by(service_id=service_id)
-        .order_by(
-            # in the order that models.notification_types are created (email, sms, letter)
-            ServiceDataRetention.notification_type
-        )
-        .all()
-    )
+    data_retention_list = ServiceDataRetention.query.filter_by(
+        service_id=service_id
+    ).order_by(
+        # in the order that models.notification_types are created (email, sms, letter)
+        ServiceDataRetention.notification_type
+    ).all()
     return data_retention_list
 
 
 def fetch_service_data_retention_by_notification_type(service_id, notification_type):
     data_retention_list = ServiceDataRetention.query.filter_by(
-        service_id=service_id, notification_type=notification_type
+        service_id=service_id,
+        notification_type=notification_type
     ).first()
     return data_retention_list
 
 
 @autocommit
 def insert_service_data_retention(service_id, notification_type, days_of_retention):
-    new_data_retention = ServiceDataRetention(
-        service_id=service_id,
-        notification_type=notification_type,
-        days_of_retention=days_of_retention,
-    )
+    new_data_retention = ServiceDataRetention(service_id=service_id,
+                                              notification_type=notification_type,
+                                              days_of_retention=days_of_retention)
 
     db.session.add(new_data_retention)
     return new_data_retention
 
 
 @autocommit
-def update_service_data_retention(
-    service_data_retention_id, service_id, days_of_retention
-):
+def update_service_data_retention(service_data_retention_id, service_id, days_of_retention):
     updated_count = ServiceDataRetention.query.filter(
         ServiceDataRetention.id == service_data_retention_id,
-        ServiceDataRetention.service_id == service_id,
-    ).update({"days_of_retention": days_of_retention, "updated_at": utc_now()})
+        ServiceDataRetention.service_id == service_id
+    ).update(
+        {
+            "days_of_retention": days_of_retention,
+            "updated_at": datetime.utcnow()
+        }
+    )
     return updated_count
 
 
-def fetch_service_data_retention_for_all_services_by_notification_type(
-    notification_type,
-):
-    return ServiceDataRetention.query.filter(
-        ServiceDataRetention.notification_type == notification_type
-    ).all()
+def fetch_service_data_retention_for_all_services_by_notification_type(notification_type):
+    return ServiceDataRetention.query.filter(ServiceDataRetention.notification_type == notification_type).all()

app/dao/service_email_reply_to_dao.py

@@ -8,31 +8,23 @@ from app.models import ServiceEmailReplyTo
 
 
 def dao_get_reply_to_by_service_id(service_id):
-    reply_to = (
-        db.session.query(ServiceEmailReplyTo)
-        .filter(
-            ServiceEmailReplyTo.service_id == service_id,
-            ServiceEmailReplyTo.archived == False,  # noqa
-        )
-        .order_by(
-            desc(ServiceEmailReplyTo.is_default), desc(ServiceEmailReplyTo.created_at)
-        )
-        .all()
-    )
+    reply_to = db.session.query(
+        ServiceEmailReplyTo
+    ).filter(
+        ServiceEmailReplyTo.service_id == service_id,
+        ServiceEmailReplyTo.archived == False  # noqa
+    ).order_by(desc(ServiceEmailReplyTo.is_default), desc(ServiceEmailReplyTo.created_at)).all()
     return reply_to
 
 
 def dao_get_reply_to_by_id(service_id, reply_to_id):
-    reply_to = (
-        db.session.query(ServiceEmailReplyTo)
-        .filter(
-            ServiceEmailReplyTo.service_id == service_id,
-            ServiceEmailReplyTo.id == reply_to_id,
-            ServiceEmailReplyTo.archived == False,  # noqa
-        )
-        .order_by(ServiceEmailReplyTo.created_at)
-        .one()
-    )
+    reply_to = db.session.query(
+        ServiceEmailReplyTo
+    ).filter(
+        ServiceEmailReplyTo.service_id == service_id,
+        ServiceEmailReplyTo.id == reply_to_id,
+        ServiceEmailReplyTo.archived == False  # noqa
+    ).order_by(ServiceEmailReplyTo.created_at).one()
     return reply_to
 
 
@@ -44,9 +36,7 @@ def add_reply_to_email_address_for_service(service_id, email_address, is_default
     else:
         _raise_when_no_default(old_default)
 
-    new_reply_to = ServiceEmailReplyTo(
-        service_id=service_id, email_address=email_address, is_default=is_default
-    )
+    new_reply_to = ServiceEmailReplyTo(service_id=service_id, email_address=email_address, is_default=is_default)
     db.session.add(new_reply_to)
     return new_reply_to
 
@@ -58,9 +48,7 @@ def update_reply_to_email_address(service_id, reply_to_id, email_address, is_def
         _reset_old_default_to_false(old_default)
     else:
         if old_default.id == reply_to_id:
-            raise InvalidRequest(
-                "You must have at least one reply to email address as the default.", 400
-            )
+            raise InvalidRequest("You must have at least one reply to email address as the default.", 400)
 
     reply_to_update = ServiceEmailReplyTo.query.get(reply_to_id)
     reply_to_update.email_address = email_address
@@ -72,13 +60,12 @@ def update_reply_to_email_address(service_id, reply_to_id, email_address, is_def
 @autocommit
 def archive_reply_to_email_address(service_id, reply_to_id):
     reply_to_archive = ServiceEmailReplyTo.query.filter_by(
-        id=reply_to_id, service_id=service_id
+        id=reply_to_id,
+        service_id=service_id
     ).one()
 
     if reply_to_archive.is_default:
-        raise ArchiveValidationError(
-            "You cannot delete a default email reply to address"
-        )
+        raise ArchiveValidationError("You cannot delete a default email reply to address")
 
     reply_to_archive.archived = True
 
@@ -95,9 +82,7 @@ def _get_existing_default(service_id):
         else:
             raise Exception(
                 "There should only be one default reply to email for each service. Service {} has {}".format(
-                    service_id, len(old_default)
-                )
-            )
+                    service_id, len(old_default)))
     return None
 
 
@@ -110,6 +95,4 @@ def _reset_old_default_to_false(old_default):
 def _raise_when_no_default(old_default):
     # check that the update is not updating the only default to false
     if not old_default:
-        raise InvalidRequest(
-            "You must have at least one reply to email address as the default.", 400
-        )
+        raise InvalidRequest("You must have at least one reply to email address as the default.", 400)

app/dao/service_guest_list_dao.py

@@ -4,8 +4,7 @@ from app.models import ServiceGuestList
 
 def dao_fetch_service_guest_list(service_id):
     return ServiceGuestList.query.filter(
-        ServiceGuestList.service_id == service_id
-    ).all()
+        ServiceGuestList.service_id == service_id).all()
 
 
 def dao_add_and_commit_guest_list_contacts(objs):
@@ -15,5 +14,4 @@ def dao_add_and_commit_guest_list_contacts(objs):
 
 def dao_remove_service_guest_list(service_id):
     return ServiceGuestList.query.filter(
-        ServiceGuestList.service_id == service_id
-    ).delete()
+        ServiceGuestList.service_id == service_id).delete()

app/dao/service_inbound_api_dao.py

@@ -1,36 +1,34 @@
+from datetime import datetime
+
 from app import create_uuid, db
 from app.dao.dao_utils import autocommit, version_class
 from app.models import ServiceInboundApi
-from app.utils import utc_now
 
 
 @autocommit
 @version_class(ServiceInboundApi)
 def save_service_inbound_api(service_inbound_api):
     service_inbound_api.id = create_uuid()
-    service_inbound_api.created_at = utc_now()
+    service_inbound_api.created_at = datetime.utcnow()
     db.session.add(service_inbound_api)
 
 
 @autocommit
 @version_class(ServiceInboundApi)
-def reset_service_inbound_api(
-    service_inbound_api, updated_by_id, url=None, bearer_token=None
-):
+def reset_service_inbound_api(service_inbound_api, updated_by_id, url=None, bearer_token=None):
     if url:
         service_inbound_api.url = url
     if bearer_token:
         service_inbound_api.bearer_token = bearer_token
     service_inbound_api.updated_by_id = updated_by_id
-    service_inbound_api.updated_at = utc_now()
+    service_inbound_api.updated_at = datetime.utcnow()
 
     db.session.add(service_inbound_api)
 
 
 def get_service_inbound_api(service_inbound_api_id, service_id):
-    return ServiceInboundApi.query.filter_by(
-        id=service_inbound_api_id, service_id=service_id
-    ).first()
+    return ServiceInboundApi.query.filter_by(id=service_inbound_api_id,
+                                             service_id=service_id).first()
 
 
 def get_service_inbound_api_for_service(service_id):

app/dao/service_permissions_dao.py

@@ -5,8 +5,7 @@ from app.models import ServicePermission
 
 def dao_fetch_service_permissions(service_id):
     return ServicePermission.query.filter(
-        ServicePermission.service_id == service_id
-    ).all()
+        ServicePermission.service_id == service_id).all()
 
 
 @autocommit
@@ -18,7 +17,6 @@ def dao_add_service_permission(service_id, permission):
 def dao_remove_service_permission(service_id, permission):
     deleted = ServicePermission.query.filter(
         ServicePermission.service_id == service_id,
-        ServicePermission.permission == permission,
-    ).delete()
+        ServicePermission.permission == permission).delete()
     db.session.commit()
     return deleted

app/dao/service_sms_sender_dao.py

@@ -10,30 +10,30 @@ def insert_service_sms_sender(service, sms_sender):
     """
     This method is called from create_service which is wrapped in a transaction.
     """
-    new_sms_sender = ServiceSmsSender(
-        sms_sender=sms_sender, service=service, is_default=True
-    )
+    new_sms_sender = ServiceSmsSender(sms_sender=sms_sender,
+                                      service=service,
+                                      is_default=True
+                                      )
     db.session.add(new_sms_sender)
 
 
 def dao_get_service_sms_senders_by_id(service_id, service_sms_sender_id):
     return ServiceSmsSender.query.filter_by(
-        id=service_sms_sender_id, service_id=service_id, archived=False
+        id=service_sms_sender_id,
+        service_id=service_id,
+        archived=False
     ).one()
 
 
 def dao_get_sms_senders_by_service_id(service_id):
-    return (
-        ServiceSmsSender.query.filter_by(service_id=service_id, archived=False)
-        .order_by(desc(ServiceSmsSender.is_default))
-        .all()
-    )
+    return ServiceSmsSender.query.filter_by(
+        service_id=service_id,
+        archived=False
+    ).order_by(desc(ServiceSmsSender.is_default)).all()
 
 
 @autocommit
-def dao_add_sms_sender_for_service(
-    service_id, sms_sender, is_default, inbound_number_id=None
-):
+def dao_add_sms_sender_for_service(service_id, sms_sender, is_default, inbound_number_id=None):
     old_default = _get_existing_default(service_id=service_id)
     if is_default:
         _reset_old_default_to_false(old_default)
@@ -44,7 +44,7 @@ def dao_add_sms_sender_for_service(
         service_id=service_id,
         sms_sender=sms_sender,
         is_default=is_default,
-        inbound_number_id=inbound_number_id,
+        inbound_number_id=inbound_number_id
     )
 
     db.session.add(new_sms_sender)
@@ -52,9 +52,7 @@ def dao_add_sms_sender_for_service(
 
 
 @autocommit
-def dao_update_service_sms_sender(
-    service_id, service_sms_sender_id, is_default, sms_sender=None
-):
+def dao_update_service_sms_sender(service_id, service_sms_sender_id, is_default, sms_sender=None):
     old_default = _get_existing_default(service_id)
     if is_default:
         _reset_old_default_to_false(old_default)
@@ -71,9 +69,7 @@ def dao_update_service_sms_sender(
 
 
 @autocommit
-def update_existing_sms_sender_with_inbound_number(
-    service_sms_sender, sms_sender, inbound_number_id
-):
+def update_existing_sms_sender_with_inbound_number(service_sms_sender, sms_sender, inbound_number_id):
     service_sms_sender.sms_sender = sms_sender
     service_sms_sender.inbound_number_id = inbound_number_id
     db.session.add(service_sms_sender)
@@ -83,7 +79,8 @@ def update_existing_sms_sender_with_inbound_number(
 @autocommit
 def archive_sms_sender(service_id, sms_sender_id):
     sms_sender_to_archive = ServiceSmsSender.query.filter_by(
-        id=sms_sender_id, service_id=service_id
+        id=sms_sender_id,
+        service_id=service_id
     ).one()
 
     if sms_sender_to_archive.inbound_number_id:
@@ -106,7 +103,8 @@ def _get_existing_default(service_id):
         else:
             raise Exception(
                 "There should only be one default sms sender for each service. Service {} has {}".format(
-                    service_id, len(old_default)
+                    service_id,
+                    len(old_default)
                 )
             )
     return None

app/dao/service_user_dao.py

@@ -1,22 +1,21 @@
+
 from app import db
 from app.dao.dao_utils import autocommit
 from app.models import ServiceUser, User
 
 
 def dao_get_service_user(user_id, service_id):
-    # TODO: This has been changed to account for the test case failure
-    # that used this method but have any service user to return. Somehow, this
-    # started to throw an error with one() method in sqlalchemy 2.0 unlike 1.4
-    return ServiceUser.query.filter_by(
-        user_id=user_id, service_id=service_id
-    ).one_or_none()
+    return ServiceUser.query.filter_by(user_id=user_id, service_id=service_id).one()
 
 
 def dao_get_active_service_users(service_id):
-    query = (
-        db.session.query(ServiceUser)
-        .join(User, User.id == ServiceUser.user_id)
-        .filter(User.state == "active", ServiceUser.service_id == service_id)
+    query = db.session.query(
+        ServiceUser
+    ).join(
+        User, User.id == ServiceUser.user_id
+    ).filter(
+        User.state == 'active',
+        ServiceUser.service_id == service_id
     )
 
     return query.all()

app/dao/services_dao.py

@@ -1,25 +1,24 @@
 import uuid
-from datetime import timedelta
+from datetime import datetime, timedelta
 
 from flask import current_app
-from sqlalchemy import Float, cast, select
+from sqlalchemy import Float, cast
 from sqlalchemy.orm import joinedload
 from sqlalchemy.sql.expression import and_, asc, case, func
 
 from app import db
 from app.dao.dao_utils import VersionOptions, autocommit, version_class
-from app.dao.date_util import generate_date_range, get_current_calendar_year
+from app.dao.date_util import get_current_calendar_year
 from app.dao.organization_dao import dao_get_organization_by_email_address
 from app.dao.service_sms_sender_dao import insert_service_sms_sender
 from app.dao.service_user_dao import dao_get_service_user
 from app.dao.template_folder_dao import dao_get_valid_template_folders_by_id
-from app.enums import (
-    KeyType,
-    NotificationStatus,
-    NotificationType,
-    ServicePermissionType,
-)
 from app.models import (
+    EMAIL_TYPE,
+    INTERNATIONAL_SMS_TYPE,
+    KEY_TYPE_TEST,
+    NOTIFICATION_PERMANENT_FAILURE,
+    SMS_TYPE,
     AnnualBilling,
     ApiKey,
     FactBilling,
@@ -27,7 +26,6 @@ from app.models import (
     InvitedUser,
     Job,
     Notification,
-    NotificationAllTimeView,
     NotificationHistory,
     Organization,
     Permission,
@@ -41,18 +39,24 @@ from app.models import (
     User,
     VerifyCode,
 )
-from app.service import statistics
 from app.utils import (
     escape_special_characters,
     get_archived_db_column_value,
     get_midnight_in_utc,
-    utc_now,
 )
 
+DEFAULT_SERVICE_PERMISSIONS = [
+    SMS_TYPE,
+    EMAIL_TYPE,
+    INTERNATIONAL_SMS_TYPE,
+]
+
 
 def dao_fetch_all_services(only_active=False):
-    query = Service.query.order_by(asc(Service.created_at)).options(
-        joinedload(Service.users)
+    query = Service.query.order_by(
+        asc(Service.created_at)
+    ).options(
+        joinedload('users')
     )
 
     if only_active:
@@ -77,92 +81,77 @@ def dao_count_live_services():
 def dao_fetch_live_services_data():
     year_start_date, year_end_date = get_current_calendar_year()
 
-    most_recent_annual_billing = (
-        db.session.query(
-            AnnualBilling.service_id,
-            func.max(AnnualBilling.financial_year_start).label("year"),
-        )
-        .group_by(AnnualBilling.service_id)
-        .subquery()
-    )
+    most_recent_annual_billing = db.session.query(
+        AnnualBilling.service_id,
+        func.max(AnnualBilling.financial_year_start).label('year')
+    ).group_by(
+        AnnualBilling.service_id
+    ).subquery()
 
     this_year_ft_billing = FactBilling.query.filter(
         FactBilling.local_date >= year_start_date,
         FactBilling.local_date <= year_end_date,
     ).subquery()
 
-    data = (
-        db.session.query(
-            Service.id.label("service_id"),
-            Service.name.label("service_name"),
-            Organization.name.label("organization_name"),
-            Organization.organization_type.label("organization_type"),
-            Service.consent_to_research.label("consent_to_research"),
-            User.name.label("contact_name"),
-            User.email_address.label("contact_email"),
-            User.mobile_number.label("contact_mobile"),
-            Service.go_live_at.label("live_date"),
-            Service.volume_sms.label("sms_volume_intent"),
-            Service.volume_email.label("email_volume_intent"),
-            case(
-                (
-                    this_year_ft_billing.c.notification_type == NotificationType.EMAIL,
-                    func.sum(this_year_ft_billing.c.notifications_sent),
-                ),
-                else_=0,
-            ).label("email_totals"),
-            case(
-                (
-                    this_year_ft_billing.c.notification_type == NotificationType.SMS,
-                    func.sum(this_year_ft_billing.c.notifications_sent),
-                ),
-                else_=0,
-            ).label("sms_totals"),
-            AnnualBilling.free_sms_fragment_limit,
+    data = db.session.query(
+        Service.id.label('service_id'),
+        Service.name.label("service_name"),
+        Organization.name.label("organization_name"),
+        Organization.organization_type.label('organization_type'),
+        Service.consent_to_research.label('consent_to_research'),
+        User.name.label('contact_name'),
+        User.email_address.label('contact_email'),
+        User.mobile_number.label('contact_mobile'),
+        Service.go_live_at.label("live_date"),
+        Service.volume_sms.label('sms_volume_intent'),
+        Service.volume_email.label('email_volume_intent'),
+        case([
+            (this_year_ft_billing.c.notification_type == 'email', func.sum(this_year_ft_billing.c.notifications_sent))
+        ], else_=0).label("email_totals"),
+        case([
+            (this_year_ft_billing.c.notification_type == 'sms', func.sum(this_year_ft_billing.c.notifications_sent))
+        ], else_=0).label("sms_totals"),
+        AnnualBilling.free_sms_fragment_limit,
+    ).join(
+        Service.annual_billing
+    ).join(
+        most_recent_annual_billing,
+        and_(
+            Service.id == most_recent_annual_billing.c.service_id,
+            AnnualBilling.financial_year_start == most_recent_annual_billing.c.year
         )
-        .join(Service.annual_billing)
-        .join(
-            most_recent_annual_billing,
-            and_(
-                Service.id == most_recent_annual_billing.c.service_id,
-                AnnualBilling.financial_year_start == most_recent_annual_billing.c.year,
-            ),
-        )
-        .outerjoin(Service.organization)
-        .outerjoin(
-            this_year_ft_billing, Service.id == this_year_ft_billing.c.service_id
-        )
-        .outerjoin(User, Service.go_live_user_id == User.id)
-        .filter(
-            Service.count_as_live.is_(True),
-            Service.active.is_(True),
-            Service.restricted.is_(False),
-        )
-        .group_by(
-            Service.id,
-            Organization.name,
-            Organization.organization_type,
-            Service.name,
-            Service.consent_to_research,
-            Service.count_as_live,
-            Service.go_live_user_id,
-            User.name,
-            User.email_address,
-            User.mobile_number,
-            Service.go_live_at,
-            Service.volume_sms,
-            Service.volume_email,
-            this_year_ft_billing.c.notification_type,
-            AnnualBilling.free_sms_fragment_limit,
-        )
-        .order_by(asc(Service.go_live_at))
-        .all()
-    )
+    ).outerjoin(
+        Service.organization
+    ).outerjoin(
+        this_year_ft_billing, Service.id == this_year_ft_billing.c.service_id
+    ).outerjoin(
+        User, Service.go_live_user_id == User.id
+    ).filter(
+        Service.count_as_live.is_(True),
+        Service.active.is_(True),
+        Service.restricted.is_(False),
+    ).group_by(
+        Service.id,
+        Organization.name,
+        Organization.organization_type,
+        Service.name,
+        Service.consent_to_research,
+        Service.count_as_live,
+        Service.go_live_user_id,
+        User.name,
+        User.email_address,
+        User.mobile_number,
+        Service.go_live_at,
+        Service.volume_sms,
+        Service.volume_email,
+        this_year_ft_billing.c.notification_type,
+        AnnualBilling.free_sms_fragment_limit,
+    ).order_by(
+        asc(Service.go_live_at)
+    ).all()
     results = []
     for row in data:
-        existing_service = next(
-            (x for x in results if x["service_id"] == row.service_id), None
-        )
+        existing_service = next((x for x in results if x['service_id'] == row.service_id), None)
 
         if existing_service is not None:
             existing_service["email_totals"] += row.email_totals
@@ -173,32 +162,38 @@ def dao_fetch_live_services_data():
 
 
 def dao_fetch_service_by_id(service_id, only_active=False):
-    stmt = (
-        select(Service)
-        .where(Service.id == service_id)
-        .options(joinedload(Service.users))
+    query = Service.query.filter_by(
+        id=service_id
+    ).options(
+        joinedload('users')
     )
 
     if only_active:
-        stmt = stmt.where(Service.active)
+        query = query.filter(Service.active)
 
-    result = db.session.execute(stmt)
-    return result.unique().scalars().one()
+    return query.one()
 
 
 def dao_fetch_service_by_inbound_number(number):
     inbound_number = InboundNumber.query.filter(
-        InboundNumber.number == number, InboundNumber.active
+        InboundNumber.number == number,
+        InboundNumber.active
     ).first()
 
     if not inbound_number:
         return None
 
-    return Service.query.filter(Service.id == inbound_number.service_id).first()
+    return Service.query.filter(
+        Service.id == inbound_number.service_id
+    ).first()
 
 
 def dao_fetch_service_by_id_with_api_keys(service_id, only_active=False):
-    query = Service.query.filter_by(id=service_id).options(joinedload(Service.api_keys))
+    query = Service.query.filter_by(
+        id=service_id
+    ).options(
+        joinedload('api_keys')
+    )
 
     if only_active:
         query = query.filter(Service.active)
@@ -207,10 +202,12 @@ def dao_fetch_service_by_id_with_api_keys(service_id, only_active=False):
 
 
 def dao_fetch_all_services_by_user(user_id, only_active=False):
-    query = (
-        Service.query.filter(Service.users.any(id=user_id))
-        .order_by(asc(Service.created_at))
-        .options(joinedload(Service.users))
+    query = Service.query.filter(
+        Service.users.any(id=user_id)
+    ).order_by(
+        asc(Service.created_at)
+    ).options(
+        joinedload('users')
     )
 
     if only_active:
@@ -220,7 +217,9 @@ def dao_fetch_all_services_by_user(user_id, only_active=False):
 
 
 def dao_fetch_all_services_created_by_user(user_id):
-    query = Service.query.filter_by(created_by_id=user_id).order_by(
+    query = Service.query.filter_by(
+        created_by_id=user_id
+    ).order_by(
         asc(Service.created_at)
     )
 
@@ -236,14 +235,11 @@ def dao_fetch_all_services_created_by_user(user_id):
 def dao_archive_service(service_id):
     # have to eager load templates and api keys so that we don't flush when we loop through them
     # to ensure that db.session still contains the models when it comes to creating history objects
-    service = (
-        Service.query.options(
-            joinedload(Service.templates).subqueryload(Template.template_redacted),
-            joinedload(Service.api_keys),
-        )
-        .filter(Service.id == service_id)
-        .one()
-    )
+    service = Service.query.options(
+        joinedload('templates'),
+        joinedload('templates.template_redacted'),
+        joinedload('api_keys'),
+    ).filter(Service.id == service_id).one()
 
     service.active = False
     service.name = get_archived_db_column_value(service.name)
@@ -255,15 +251,16 @@ def dao_archive_service(service_id):
 
     for api_key in service.api_keys:
         if not api_key.expiry_date:
-            api_key.expiry_date = utc_now()
+            api_key.expiry_date = datetime.utcnow()
 
 
 def dao_fetch_service_by_id_and_user(service_id, user_id):
-    return (
-        Service.query.filter(Service.users.any(id=user_id), Service.id == service_id)
-        .options(joinedload(Service.users))
-        .one()
-    )
+    return Service.query.filter(
+        Service.users.any(id=user_id),
+        Service.id == service_id
+    ).options(
+        joinedload('users')
+    ).one()
 
 
 @autocommit
@@ -274,31 +271,28 @@ def dao_create_service(
     service_id=None,
     service_permissions=None,
 ):
+
     if not user:
         raise ValueError("Can't create a service without a user")
 
     if service_permissions is None:
-        service_permissions = ServicePermissionType.defaults()
+        service_permissions = DEFAULT_SERVICE_PERMISSIONS
 
     organization = dao_get_organization_by_email_address(user.email_address)
 
     from app.dao.permissions_dao import permission_dao
-
     service.users.append(user)
     permission_dao.add_default_service_permissions_for_user(user, service)
-    service.id = (
-        service_id or uuid.uuid4()
-    )  # must be set now so version history model can use same id
+    service.id = service_id or uuid.uuid4()  # must be set now so version history model can use same id
     service.active = True
+    service.research_mode = False
 
     for permission in service_permissions:
-        service_permission = ServicePermission(
-            service_id=service.id, permission=permission
-        )
+        service_permission = ServicePermission(service_id=service.id, permission=permission)
         service.permissions.append(service_permission)
 
     # do we just add the default - or will we get a value from FE?
-    insert_service_sms_sender(service, current_app.config["FROM_NUMBER"])
+    insert_service_sms_sender(service, current_app.config['FROM_NUMBER'])
 
     if organization:
         service.organization_id = organization.id
@@ -324,22 +318,12 @@ def dao_add_user_to_service(service, user, permissions=None, folder_permissions=
 
     try:
         from app.dao.permissions_dao import permission_dao
-
-        # As per SQLAlchemy 2.0, we need to add the user to the service only if the user is not already added;
-        # otherwise it throws sqlalchemy.exc.IntegrityError:
-        # (psycopg2.errors.UniqueViolation) duplicate key value violates unique constraint "uix_user_to_service"
-        service_user = dao_get_service_user(user.id, service.id)
-        if service_user is None:
-            service.users.append(user)
-        permission_dao.set_user_service_permission(
-            user, service, permissions, _commit=False
-        )
+        service.users.append(user)
+        permission_dao.set_user_service_permission(user, service, permissions, _commit=False)
         db.session.add(service)
 
         service_user = dao_get_service_user(user.id, service.id)
-        valid_template_folders = dao_get_valid_template_folders_by_id(
-            folder_permissions
-        )
+        valid_template_folders = dao_get_valid_template_folders_by_id(folder_permissions)
         service_user.folders = valid_template_folders
         db.session.add(service_user)
 
@@ -353,7 +337,6 @@ def dao_add_user_to_service(service, user, permissions=None, folder_permissions=
 def dao_remove_user_from_service(service, user):
     try:
         from app.dao.permissions_dao import permission_dao
-
         permission_dao.remove_user_service_permissions(user, service)
 
         service_user = dao_get_service_user(user.id, service.id)
@@ -366,14 +349,13 @@ def dao_remove_user_from_service(service, user):
 
 
 def delete_service_and_all_associated_db_objects(service):
+
     def _delete_commit(query):
         query.delete(synchronize_session=False)
         db.session.commit()
 
     subq = db.session.query(Template.id).filter_by(service=service).subquery()
-    _delete_commit(
-        TemplateRedacted.query.filter(TemplateRedacted.template_id.in_(subq))
-    )
+    _delete_commit(TemplateRedacted.query.filter(TemplateRedacted.template_id.in_(subq)))
 
     _delete_commit(ServiceSmsSender.query.filter_by(service=service))
     _delete_commit(ServiceEmailReplyTo.query.filter_by(service=service))
@@ -389,9 +371,7 @@ def delete_service_and_all_associated_db_objects(service):
     _delete_commit(ApiKey.get_history_model().query.filter_by(service_id=service.id))
     _delete_commit(AnnualBilling.query.filter_by(service_id=service.id))
 
-    verify_codes = VerifyCode.query.join(User).filter(
-        User.id.in_([x.id for x in service.users])
-    )
+    verify_codes = VerifyCode.query.join(User).filter(User.id.in_([x.id for x in service.users]))
     list(map(db.session.delete, verify_codes))
     db.session.commit()
     users = [x for x in service.users]
@@ -407,123 +387,60 @@ def delete_service_and_all_associated_db_objects(service):
 
 
 def dao_fetch_todays_stats_for_service(service_id):
-    today = utc_now().date()
+    today = datetime.utcnow().date()
     start_date = get_midnight_in_utc(today)
-    return (
-        db.session.query(
-            Notification.notification_type,
-            Notification.status,
-            func.count(Notification.id).label("count"),
-        )
-        .filter(
-            Notification.service_id == service_id,
-            Notification.key_type != KeyType.TEST,
-            Notification.created_at >= start_date,
-        )
-        .group_by(
-            Notification.notification_type,
-            Notification.status,
-        )
-        .all()
-    )
+    return db.session.query(
+        Notification.notification_type,
+        Notification.status,
+        func.count(Notification.id).label('count')
+    ).filter(
+        Notification.service_id == service_id,
+        Notification.key_type != KEY_TYPE_TEST,
+        Notification.created_at >= start_date
+    ).group_by(
+        Notification.notification_type,
+        Notification.status,
+    ).all()
 
 
-def dao_fetch_stats_for_service_from_days(service_id, start_date, end_date):
-    start_date = get_midnight_in_utc(start_date)
-    end_date = get_midnight_in_utc(end_date + timedelta(days=1))
-
-    return (
-        db.session.query(
-            NotificationAllTimeView.notification_type,
-            NotificationAllTimeView.status,
-            func.date_trunc("day", NotificationAllTimeView.created_at).label("day"),
-            func.count(NotificationAllTimeView.id).label("count"),
-        )
-        .filter(
-            NotificationAllTimeView.service_id == service_id,
-            NotificationAllTimeView.key_type != KeyType.TEST,
-            NotificationAllTimeView.created_at >= start_date,
-            NotificationAllTimeView.created_at < end_date,
-        )
-        .group_by(
-            NotificationAllTimeView.notification_type,
-            NotificationAllTimeView.status,
-            func.date_trunc("day", NotificationAllTimeView.created_at),
-        )
-        .all()
-    )
-
-
-def dao_fetch_stats_for_service_from_days_for_user(
-    service_id, start_date, end_date, user_id
-):
-    start_date = get_midnight_in_utc(start_date)
-    end_date = get_midnight_in_utc(end_date + timedelta(days=1))
-
-    return (
-        db.session.query(
-            NotificationAllTimeView.notification_type,
-            NotificationAllTimeView.status,
-            func.date_trunc("day", NotificationAllTimeView.created_at).label("day"),
-            func.count(NotificationAllTimeView.id).label("count"),
-        )
-        .filter(
-            NotificationAllTimeView.service_id == service_id,
-            NotificationAllTimeView.key_type != KeyType.TEST,
-            NotificationAllTimeView.created_at >= start_date,
-            NotificationAllTimeView.created_at < end_date,
-            NotificationAllTimeView.created_by_id == user_id,
-        )
-        .group_by(
-            NotificationAllTimeView.notification_type,
-            NotificationAllTimeView.status,
-            func.date_trunc("day", NotificationAllTimeView.created_at),
-        )
-        .all()
-    )
-
-
-def dao_fetch_todays_stats_for_all_services(
-    include_from_test_key=True, only_active=True
-):
-    today = utc_now().date()
+def dao_fetch_todays_stats_for_all_services(include_from_test_key=True, only_active=True):
+    today = datetime.utcnow().date()
     start_date = get_midnight_in_utc(today)
     end_date = get_midnight_in_utc(today + timedelta(days=1))
 
-    subquery = (
-        db.session.query(
-            Notification.notification_type,
-            Notification.status,
-            Notification.service_id,
-            func.count(Notification.id).label("count"),
-        )
-        .filter(
-            Notification.created_at >= start_date, Notification.created_at < end_date
-        )
-        .group_by(
-            Notification.notification_type, Notification.status, Notification.service_id
-        )
+    subquery = db.session.query(
+        Notification.notification_type,
+        Notification.status,
+        Notification.service_id,
+        func.count(Notification.id).label('count')
+    ).filter(
+        Notification.created_at >= start_date,
+        Notification.created_at < end_date
+    ).group_by(
+        Notification.notification_type,
+        Notification.status,
+        Notification.service_id
     )
 
     if not include_from_test_key:
-        subquery = subquery.filter(Notification.key_type != KeyType.TEST)
+        subquery = subquery.filter(Notification.key_type != KEY_TYPE_TEST)
 
     subquery = subquery.subquery()
 
-    query = (
-        db.session.query(
-            Service.id.label("service_id"),
-            Service.name,
-            Service.restricted,
-            Service.active,
-            Service.created_at,
-            subquery.c.notification_type,
-            subquery.c.status,
-            subquery.c.count,
-        )
-        .outerjoin(subquery, subquery.c.service_id == Service.id)
-        .order_by(Service.id)
-    )
+    query = db.session.query(
+        Service.id.label('service_id'),
+        Service.name,
+        Service.restricted,
+        Service.research_mode,
+        Service.active,
+        Service.created_at,
+        subquery.c.notification_type,
+        subquery.c.status,
+        subquery.c.count
+    ).outerjoin(
+        subquery,
+        subquery.c.service_id == Service.id
+    ).order_by(Service.id)
 
     if only_active:
         query = query.filter(Service.active)
@@ -539,17 +456,13 @@ def dao_fetch_todays_stats_for_all_services(
 def dao_suspend_service(service_id):
     # have to eager load api keys so that we don't flush when we loop through them
     # to ensure that db.session still contains the models when it comes to creating history objects
-    service = (
-        Service.query.options(
-            joinedload(Service.api_keys),
-        )
-        .filter(Service.id == service_id)
-        .one()
-    )
+    service = Service.query.options(
+        joinedload('api_keys'),
+    ).filter(Service.id == service_id).one()
 
     for api_key in service.api_keys:
         if not api_key.expiry_date:
-            api_key.expiry_date = utc_now()
+            api_key.expiry_date = datetime.utcnow()
 
     service.active = False
 
@@ -562,154 +475,99 @@ def dao_resume_service(service_id):
 
 
 def dao_fetch_active_users_for_service(service_id):
-    query = User.query.filter(User.services.any(id=service_id), User.state == "active")
+    query = User.query.filter(
+        User.services.any(id=service_id),
+        User.state == 'active'
+    )
 
     return query.all()
 
 
 def dao_find_services_sending_to_tv_numbers(start_date, end_date, threshold=500):
-    return (
-        db.session.query(
-            Notification.service_id.label("service_id"),
-            func.count(Notification.id).label("notification_count"),
-        )
-        .filter(
-            Notification.service_id == Service.id,
-            Notification.created_at >= start_date,
-            Notification.created_at <= end_date,
-            Notification.key_type != KeyType.TEST,
-            Notification.notification_type == NotificationType.SMS,
-            func.substr(Notification.normalised_to, 3, 7) == "7700900",
-            Service.restricted == False,  # noqa
-            Service.active == True,  # noqa
-        )
-        .group_by(
-            Notification.service_id,
-        )
-        .having(func.count(Notification.id) > threshold)
-        .all()
-    )
+    return db.session.query(
+        Notification.service_id.label('service_id'),
+        func.count(Notification.id).label('notification_count')
+    ).filter(
+        Notification.service_id == Service.id,
+        Notification.created_at >= start_date,
+        Notification.created_at <= end_date,
+        Notification.key_type != KEY_TYPE_TEST,
+        Notification.notification_type == SMS_TYPE,
+        func.substr(Notification.normalised_to, 3, 7) == '7700900',
+        Service.restricted == False,  # noqa
+        Service.research_mode == False,  # noqa
+        Service.active == True,  # noqa
+    ).group_by(
+        Notification.service_id,
+    ).having(
+        func.count(Notification.id) > threshold
+    ).all()
 
 
 def dao_find_services_with_high_failure_rates(start_date, end_date, threshold=10000):
-    subquery = (
-        db.session.query(
-            func.count(Notification.id).label("total_count"),
-            Notification.service_id.label("service_id"),
-        )
-        .filter(
-            Notification.service_id == Service.id,
-            Notification.created_at >= start_date,
-            Notification.created_at <= end_date,
-            Notification.key_type != KeyType.TEST,
-            Notification.notification_type == NotificationType.SMS,
-            Service.restricted == False,  # noqa
-            Service.active == True,  # noqa
-        )
-        .group_by(
-            Notification.service_id,
-        )
-        .having(func.count(Notification.id) >= threshold)
+    subquery = db.session.query(
+        func.count(Notification.id).label('total_count'),
+        Notification.service_id.label('service_id')
+    ).filter(
+        Notification.service_id == Service.id,
+        Notification.created_at >= start_date,
+        Notification.created_at <= end_date,
+        Notification.key_type != KEY_TYPE_TEST,
+        Notification.notification_type == SMS_TYPE,
+        Service.restricted == False,  # noqa
+        Service.research_mode == False,  # noqa
+        Service.active == True,  # noqa
+    ).group_by(
+        Notification.service_id,
+    ).having(
+        func.count(Notification.id) >= threshold
     )
 
     subquery = subquery.subquery()
 
-    query = (
-        db.session.query(
-            Notification.service_id.label("service_id"),
-            func.count(Notification.id).label("permanent_failure_count"),
-            subquery.c.total_count.label("total_count"),
-            (
-                cast(func.count(Notification.id), Float)
-                / cast(subquery.c.total_count, Float)
-            ).label("permanent_failure_rate"),
-        )
-        .join(subquery, subquery.c.service_id == Notification.service_id)
-        .filter(
-            Notification.service_id == Service.id,
-            Notification.created_at >= start_date,
-            Notification.created_at <= end_date,
-            Notification.key_type != KeyType.TEST,
-            Notification.notification_type == NotificationType.SMS,
-            Notification.status == NotificationStatus.PERMANENT_FAILURE,
-            Service.restricted == False,  # noqa
-            Service.active == True,  # noqa
-        )
-        .group_by(Notification.service_id, subquery.c.total_count)
-        .having(
-            cast(func.count(Notification.id), Float)
-            / cast(subquery.c.total_count, Float)
-            >= 0.25
-        )
+    query = db.session.query(
+        Notification.service_id.label('service_id'),
+        func.count(Notification.id).label('permanent_failure_count'),
+        subquery.c.total_count.label('total_count'),
+        (cast(func.count(Notification.id), Float) / cast(subquery.c.total_count, Float)).label('permanent_failure_rate')
+    ).join(
+        subquery,
+        subquery.c.service_id == Notification.service_id
+    ).filter(
+        Notification.service_id == Service.id,
+        Notification.created_at >= start_date,
+        Notification.created_at <= end_date,
+        Notification.key_type != KEY_TYPE_TEST,
+        Notification.notification_type == SMS_TYPE,
+        Notification.status == NOTIFICATION_PERMANENT_FAILURE,
+        Service.restricted == False,  # noqa
+        Service.research_mode == False,  # noqa
+        Service.active == True,  # noqa
+    ).group_by(
+        Notification.service_id,
+        subquery.c.total_count
+    ).having(
+        cast(func.count(Notification.id), Float) / cast(subquery.c.total_count, Float) >= 0.25
     )
 
     return query.all()
 
 
 def get_live_services_with_organization():
-    query = (
-        db.session.query(
-            Service.id.label("service_id"),
-            Service.name.label("service_name"),
-            Organization.id.label("organization_id"),
-            Organization.name.label("organization_name"),
-        )
-        .outerjoin(Service.organization)
-        .filter(
-            Service.count_as_live.is_(True),
-            Service.active.is_(True),
-            Service.restricted.is_(False),
-        )
-        .order_by(Organization.name, Service.name)
+    query = db.session.query(
+        Service.id.label("service_id"),
+        Service.name.label("service_name"),
+        Organization.id.label("organization_id"),
+        Organization.name.label("organization_name")
+    ).outerjoin(
+        Service.organization
+    ).filter(
+        Service.count_as_live.is_(True),
+        Service.active.is_(True),
+        Service.restricted.is_(False)
+    ).order_by(
+        Organization.name,
+        Service.name
     )
 
     return query.all()
-
-
-def fetch_notification_stats_for_service_by_month_by_user(
-    start_date, end_date, service_id, user_id
-):
-    return (
-        db.session.query(
-            func.date_trunc("month", NotificationAllTimeView.created_at).label("month"),
-            NotificationAllTimeView.notification_type,
-            (NotificationAllTimeView.status).label("notification_status"),
-            func.count(NotificationAllTimeView.id).label("count"),
-        )
-        .filter(
-            NotificationAllTimeView.service_id == service_id,
-            NotificationAllTimeView.created_at >= start_date,
-            NotificationAllTimeView.created_at < end_date,
-            NotificationAllTimeView.key_type != KeyType.TEST,
-            NotificationAllTimeView.created_by_id == user_id,
-        )
-        .group_by(
-            func.date_trunc("month", NotificationAllTimeView.created_at).label("month"),
-            NotificationAllTimeView.notification_type,
-            NotificationAllTimeView.status,
-        )
-        .all()
-    )
-
-
-def get_specific_days_stats(data, start_date, days=None, end_date=None):
-    if days is not None and end_date is not None:
-        raise ValueError("Only set days OR set end_date, not both.")
-    elif days is not None:
-        gen_range = generate_date_range(start_date, days=days)
-    elif end_date is not None:
-        gen_range = generate_date_range(start_date, end_date)
-    else:
-        raise ValueError("Either days or end_date must be set.")
-
-    grouped_data = {date: [] for date in gen_range} | {
-        day: [row for row in data if row.day.date() == day]
-        for day in {item.day.date() for item in data}
-    }
-
-    stats = {
-        day.strftime("%Y-%m-%d"): statistics.format_statistics(rows)
-        for day, rows in grouped_data.items()
-    }
-
-    return stats

app/dao/template_folder_dao.py

@@ -5,7 +5,8 @@ from app.models import TemplateFolder
 
 def dao_get_template_folder_by_id_and_service_id(template_folder_id, service_id):
     return TemplateFolder.query.filter(
-        TemplateFolder.id == template_folder_id, TemplateFolder.service_id == service_id
+        TemplateFolder.id == template_folder_id,
+        TemplateFolder.service_id == service_id
     ).one()
 
 

app/dao/templates_dao.py

@@ -1,19 +1,19 @@
 import uuid
+from datetime import datetime
 
 from sqlalchemy import asc, desc
 
 from app import db
 from app.dao.dao_utils import VersionOptions, autocommit, version_class
 from app.models import Template, TemplateHistory, TemplateRedacted
-from app.utils import utc_now
 
 
 @autocommit
-@version_class(VersionOptions(Template, history_class=TemplateHistory))
+@version_class(
+    VersionOptions(Template, history_class=TemplateHistory)
+)
 def dao_create_template(template):
-    template.id = (
-        uuid.uuid4()
-    )  # must be set now so version history model can use same id
+    template.id = uuid.uuid4()  # must be set now so version history model can use same id
     template.archived = False
 
     redacted_dict = {
@@ -31,7 +31,9 @@ def dao_create_template(template):
 
 
 @autocommit
-@version_class(VersionOptions(Template, history_class=TemplateHistory))
+@version_class(
+    VersionOptions(Template, history_class=TemplateHistory)
+)
 def dao_update_template(template):
     db.session.add(template)
 
@@ -39,7 +41,7 @@ def dao_update_template(template):
 @autocommit
 def dao_redact_template(template, user_id):
     template.template_redacted.redact_personalisation = True
-    template.template_redacted.updated_at = utc_now()
+    template.template_redacted.updated_at = datetime.utcnow()
     template.template_redacted.updated_by_id = user_id
     db.session.add(template.template_redacted)
 
@@ -47,52 +49,47 @@ def dao_redact_template(template, user_id):
 def dao_get_template_by_id_and_service_id(template_id, service_id, version=None):
     if version is not None:
         return TemplateHistory.query.filter_by(
-            id=template_id, hidden=False, service_id=service_id, version=version
-        ).one()
-    return Template.query.filter_by(
-        id=template_id, hidden=False, service_id=service_id
-    ).one()
+            id=template_id,
+            hidden=False,
+            service_id=service_id,
+            version=version).one()
+    return Template.query.filter_by(id=template_id, hidden=False, service_id=service_id).one()
 
 
 def dao_get_template_by_id(template_id, version=None):
     if version is not None:
-        return TemplateHistory.query.filter_by(id=template_id, version=version).one()
+        return TemplateHistory.query.filter_by(
+            id=template_id,
+            version=version).one()
     return Template.query.filter_by(id=template_id).one()
 
 
 def dao_get_all_templates_for_service(service_id, template_type=None):
     if template_type is not None:
-        return (
-            Template.query.filter_by(
-                service_id=service_id,
-                template_type=template_type,
-                hidden=False,
-                archived=False,
-            )
-            .order_by(
-                asc(Template.name),
-                asc(Template.template_type),
-            )
-            .all()
-        )
-
-    return (
-        Template.query.filter_by(service_id=service_id, hidden=False, archived=False)
-        .order_by(
+        return Template.query.filter_by(
+            service_id=service_id,
+            template_type=template_type,
+            hidden=False,
+            archived=False
+        ).order_by(
             asc(Template.name),
             asc(Template.template_type),
-        )
-        .all()
-    )
+        ).all()
+
+    return Template.query.filter_by(
+        service_id=service_id,
+        hidden=False,
+        archived=False
+    ).order_by(
+        asc(Template.name),
+        asc(Template.template_type),
+    ).all()
 
 
 def dao_get_template_versions(service_id, template_id):
-    return (
-        TemplateHistory.query.filter_by(
-            service_id=service_id,
-            id=template_id,
-            hidden=False,
-        )
-        .order_by(desc(TemplateHistory.version))
-        .all()
-    )
+    return TemplateHistory.query.filter_by(
+        service_id=service_id, id=template_id,
+        hidden=False,
+    ).order_by(
+        desc(TemplateHistory.version)
+    ).all()

app/dao/uploads_dao.py

@@ -1,25 +1,31 @@
+from datetime import datetime
 from os import getenv
 
 from flask import current_app
 from sqlalchemy import String, and_, desc, func, literal, text
 
 from app import db
-from app.enums import JobStatus, NotificationStatus, NotificationType
-from app.models import Job, Notification, ServiceDataRetention, Template
-from app.utils import midnight_n_days_ago, utc_now
+from app.models import (
+    JOB_STATUS_CANCELLED,
+    JOB_STATUS_SCHEDULED,
+    LETTER_TYPE,
+    NOTIFICATION_CANCELLED,
+    Job,
+    Notification,
+    ServiceDataRetention,
+    Template,
+)
+from app.utils import midnight_n_days_ago
 
 
 def _get_printing_day(created_at):
     return func.date_trunc(
-        "day",
-        func.timezone(
-            getenv("TIMEZONE", "America/New_York"), func.timezone("UTC", created_at)
-        )
-        + text(
+        'day',
+        func.timezone(getenv("TIMEZONE", "America/New_York"), func.timezone('UTC', created_at)) + text(
             # We add 6 hours 30 minutes to the local created_at time so that
             # any letters created after 5:30pm get shifted into the next day
             "interval '6 hours 30 minutes'"
-        ),
+        )
     )
 
 
@@ -31,108 +37,94 @@ def _get_printing_datetime(created_at):
 
 
 def _naive_gmt_to_utc(column):
-    return func.timezone(
-        "UTC", func.timezone(getenv("TIMEZONE", "America/New_York"), column)
-    )
+    return func.timezone('UTC', func.timezone(getenv("TIMEZONE", "America/New_York"), column))
 
 
 def dao_get_uploads_by_service_id(service_id, limit_days=None, page=1, page_size=50):
     # Hardcoded filter to exclude cancelled or scheduled jobs
     # for the moment, but we may want to change this method take 'statuses' as a argument in the future
-    today = utc_now().date()
+    today = datetime.utcnow().date()
     jobs_query_filter = [
         Job.service_id == service_id,
-        Job.original_file_name != current_app.config["TEST_MESSAGE_FILENAME"],
-        Job.original_file_name != current_app.config["ONE_OFF_MESSAGE_FILENAME"],
-        Job.job_status.notin_([JobStatus.CANCELLED, JobStatus.SCHEDULED]),
-        func.coalesce(Job.processing_started, Job.created_at)
-        >= today - func.coalesce(ServiceDataRetention.days_of_retention, 7),
+        Job.original_file_name != current_app.config['TEST_MESSAGE_FILENAME'],
+        Job.original_file_name != current_app.config['ONE_OFF_MESSAGE_FILENAME'],
+        Job.job_status.notin_([JOB_STATUS_CANCELLED, JOB_STATUS_SCHEDULED]),
+        func.coalesce(
+            Job.processing_started, Job.created_at
+        ) >= today - func.coalesce(ServiceDataRetention.days_of_retention, 7),
     ]
     if limit_days is not None:
         jobs_query_filter.append(Job.created_at >= midnight_n_days_ago(limit_days))
 
-    jobs_query = (
-        db.session.query(
-            Job.id,
-            Job.original_file_name,
-            Job.notification_count,
-            Template.template_type,
-            func.coalesce(ServiceDataRetention.days_of_retention, 7).label(
-                "days_of_retention"
-            ),
-            Job.created_at.label("created_at"),
-            Job.scheduled_for.label("scheduled_for"),
-            Job.processing_started.label("processing_started"),
-            Job.job_status.label("status"),
-            literal("job").label("upload_type"),
-            literal(None).label("recipient"),
+    jobs_query = db.session.query(
+        Job.id,
+        Job.original_file_name,
+        Job.notification_count,
+        Template.template_type,
+        func.coalesce(ServiceDataRetention.days_of_retention, 7).label('days_of_retention'),
+        Job.created_at.label("created_at"),
+        Job.scheduled_for.label("scheduled_for"),
+        Job.processing_started.label('processing_started'),
+        Job.job_status.label("status"),
+        literal('job').label('upload_type'),
+        literal(None).label('recipient'),
+    ).join(
+        Template, Job.template_id == Template.id
+    ).outerjoin(
+        ServiceDataRetention, and_(
+            Template.service_id == ServiceDataRetention.service_id,
+            func.cast(Template.template_type, String) == func.cast(ServiceDataRetention.notification_type, String)
         )
-        .join(Template, Job.template_id == Template.id)
-        .outerjoin(
-            ServiceDataRetention,
-            and_(
-                Template.service_id == ServiceDataRetention.service_id,
-                func.cast(Template.template_type, String)
-                == func.cast(ServiceDataRetention.notification_type, String),
-            ),
-        )
-        .filter(*jobs_query_filter)
+    ).filter(
+        *jobs_query_filter
     )
 
     letters_query_filter = [
         Notification.service_id == service_id,
-        Notification.notification_type == NotificationType.LETTER,
+        Notification.notification_type == LETTER_TYPE,
         Notification.api_key_id == None,  # noqa
-        Notification.status != NotificationStatus.CANCELLED,
+        Notification.status != NOTIFICATION_CANCELLED,
         Template.hidden == True,  # noqa
-        Notification.created_at
-        >= today - func.coalesce(ServiceDataRetention.days_of_retention, 7),
+        Notification.created_at >= today - func.coalesce(ServiceDataRetention.days_of_retention, 7)
     ]
     if limit_days is not None:
-        letters_query_filter.append(
-            Notification.created_at >= midnight_n_days_ago(limit_days)
-        )
+        letters_query_filter.append(Notification.created_at >= midnight_n_days_ago(limit_days))
 
-    letters_subquery = (
-        db.session.query(
-            func.count().label("notification_count"),
-            _naive_gmt_to_utc(_get_printing_datetime(Notification.created_at)).label(
-                "printing_at"
-            ),
+    letters_subquery = db.session.query(
+        func.count().label('notification_count'),
+        _naive_gmt_to_utc(_get_printing_datetime(Notification.created_at)).label('printing_at'),
+    ).join(
+        Template, Notification.template_id == Template.id
+    ).outerjoin(
+        ServiceDataRetention, and_(
+            Template.service_id == ServiceDataRetention.service_id,
+            func.cast(Template.template_type, String) == func.cast(ServiceDataRetention.notification_type, String)
         )
-        .join(Template, Notification.template_id == Template.id)
-        .outerjoin(
-            ServiceDataRetention,
-            and_(
-                Template.service_id == ServiceDataRetention.service_id,
-                func.cast(Template.template_type, String)
-                == func.cast(ServiceDataRetention.notification_type, String),
-            ),
-        )
-        .filter(*letters_query_filter)
-        .group_by("printing_at")
-        .subquery()
-    )
+    ).filter(
+        *letters_query_filter
+    ).group_by(
+        'printing_at'
+    ).subquery()
 
     letters_query = db.session.query(
-        literal(None).label("id"),
-        literal("Uploaded letters").label("original_file_name"),
-        letters_subquery.c.notification_count.label("notification_count"),
-        literal("letter").label("template_type"),
-        literal(None).label("days_of_retention"),
-        letters_subquery.c.printing_at.label("created_at"),
-        literal(None).label("scheduled_for"),
-        letters_subquery.c.printing_at.label("processing_started"),
-        literal(None).label("status"),
-        literal("letter_day").label("upload_type"),
-        literal(None).label("recipient"),
+        literal(None).label('id'),
+        literal('Uploaded letters').label('original_file_name'),
+        letters_subquery.c.notification_count.label('notification_count'),
+        literal('letter').label('template_type'),
+        literal(None).label('days_of_retention'),
+        letters_subquery.c.printing_at.label('created_at'),
+        literal(None).label('scheduled_for'),
+        letters_subquery.c.printing_at.label('processing_started'),
+        literal(None).label('status'),
+        literal('letter_day').label('upload_type'),
+        literal(None).label('recipient'),
     ).group_by(
         letters_subquery.c.notification_count,
         letters_subquery.c.printing_at,
     )
 
-    return (
-        jobs_query.union_all(letters_query)
-        .order_by(desc("processing_started"), desc("created_at"))
-        .paginate(page=page, per_page=page_size)
-    )
+    return jobs_query.union_all(
+        letters_query
+    ).order_by(
+        desc("processing_started"), desc("created_at")
+    ).paginate(page=page, per_page=page_size)

app/dao/users_dao.py

@@ -1,20 +1,17 @@
 import uuid
-from datetime import timedelta
+from datetime import datetime, timedelta
 from secrets import randbelow
 
-import sqlalchemy
-from flask import current_app
-from sqlalchemy import func, text
+from sqlalchemy import func
 from sqlalchemy.orm import joinedload
 
 from app import db
 from app.dao.dao_utils import autocommit
 from app.dao.permissions_dao import permission_dao
 from app.dao.service_user_dao import dao_get_service_users_by_user_id
-from app.enums import AuthType, PermissionType
 from app.errors import InvalidRequest
-from app.models import Organization, Service, User, VerifyCode
-from app.utils import escape_special_characters, get_archived_db_column_value, utc_now
+from app.models import EMAIL_AUTH_TYPE, User, VerifyCode
+from app.utils import escape_special_characters, get_archived_db_column_value
 
 
 def _remove_values_for_keys_if_present(dict, keys):
@@ -23,64 +20,23 @@ def _remove_values_for_keys_if_present(dict, keys):
 
 
 def create_secret_code(length=6):
-    random_number = randbelow(10**length)
+    random_number = randbelow(10 ** length)
     return "{:0{length}d}".format(random_number, length=length)
 
 
-def get_login_gov_user(login_uuid, email_address):
-    """
-    We want to check to see if the user is registered with login.gov
-    If we can find the login.gov uuid in our user table, then they are.
-
-    Also, because we originally keyed off email address we might have a few
-    older users who registered with login.gov but we don't know what their
-    login.gov uuids are.  Eventually the code that checks by email address
-    should be removed.
-    """
-
-    user = User.query.filter_by(login_uuid=login_uuid).first()
-    if user:
-        if user.email_address != email_address:
-            try:
-                save_user_attribute(user, {"email_address": email_address})
-            except sqlalchemy.exc.IntegrityError as ie:
-                # We are trying to change the email address as a courtesy,
-                # based on the assumption that the user has somehow changed their
-                # address in login.gov.
-                # But if we cannot change the email address, at least we don't
-                # want to fail here, otherwise the user will be locked out.
-                current_app.logger.error(ie)
-                db.session.rollback()
-
-        return user
-    # Remove this 1 July 2025, all users should have login.gov uuids by now
-    user = User.query.filter(User.email_address.ilike(email_address)).first()
-
-    if user:
-        save_user_attribute(user, {"login_uuid": login_uuid})
-        return user
-
-    return None
-
-
 def save_user_attribute(usr, update_dict=None):
     db.session.query(User).filter_by(id=usr.id).update(update_dict or {})
     db.session.commit()
 
 
-def save_model_user(
-    user,
-    update_dict=None,
-    password=None,
-    validated_email_access=False,
-):
+def save_model_user(user, update_dict=None, password=None, validated_email_access=False):
     if password:
         user.password = password
-        user.password_changed_at = utc_now()
+        user.password_changed_at = datetime.utcnow()
     if validated_email_access:
-        user.email_access_validated_at = utc_now()
+        user.email_access_validated_at = datetime.utcnow()
     if update_dict:
-        _remove_values_for_keys_if_present(update_dict, ["id", "password_changed_at"])
+        _remove_values_for_keys_if_present(update_dict, ['id', 'password_changed_at'])
         db.session.query(User).filter_by(id=user.id).update(update_dict or {})
     else:
         db.session.add(user)
@@ -88,11 +44,9 @@ def save_model_user(
 
 
 def create_user_code(user, code, code_type):
-    verify_code = VerifyCode(
-        code_type=code_type,
-        expiry_datetime=utc_now() + timedelta(minutes=30),
-        user=user,
-    )
+    verify_code = VerifyCode(code_type=code_type,
+                             expiry_datetime=datetime.utcnow() + timedelta(minutes=30),
+                             user=user)
     verify_code.code = code
     db.session.add(verify_code)
     db.session.commit()
@@ -102,18 +56,16 @@ def create_user_code(user, code, code_type):
 def get_user_code(user, code, code_type):
     # Get the most recent codes to try and reduce the
     # time searching for the correct code.
-    codes = VerifyCode.query.filter_by(user=user, code_type=code_type).order_by(
-        VerifyCode.created_at.desc()
-    )
+    codes = VerifyCode.query.filter_by(
+        user=user, code_type=code_type).order_by(
+        VerifyCode.created_at.desc())
     return next((x for x in codes if x.check_code(code)), None)
 
 
 def delete_codes_older_created_more_than_a_day_ago():
-    deleted = (
-        db.session.query(VerifyCode)
-        .filter(VerifyCode.created_at < utc_now() - timedelta(hours=24))
-        .delete()
-    )
+    deleted = db.session.query(VerifyCode).filter(
+        VerifyCode.created_at < datetime.utcnow() - timedelta(hours=24)
+    ).delete()
     db.session.commit()
     return deleted
 
@@ -138,8 +90,8 @@ def delete_user_verify_codes(user):
 def count_user_verify_codes(user):
     query = VerifyCode.query.filter(
         VerifyCode.user == user,
-        VerifyCode.expiry_datetime > utc_now(),
-        VerifyCode.code_used.is_(False),
+        VerifyCode.expiry_datetime > datetime.utcnow(),
+        VerifyCode.code_used.is_(False)
     )
     return query.count()
 
@@ -150,10 +102,6 @@ def get_user_by_id(user_id=None):
     return User.query.filter_by().all()
 
 
-def get_users():
-    return User.query.all()
-
-
 def get_user_by_email(email):
     return User.query.filter(func.lower(User.email_address) == func.lower(email)).one()
 
@@ -179,25 +127,22 @@ def reset_failed_login_count(user):
 def update_user_password(user, password):
     # reset failed login count - they've just reset their password so should be fine
     user.password = password
-    user.password_changed_at = utc_now()
+    user.password_changed_at = datetime.utcnow()
     db.session.add(user)
     db.session.commit()
 
 
 def get_user_and_accounts(user_id):
-    # TODO: With sqlalchemy 2.0 change as below because of the breaking change
-    # at User.organizations.services, we need to verify that the below subqueryload
-    # that we have put is functionally doing the same thing as before
-    return (
-        User.query.filter(User.id == user_id)
-        .options(
-            # eagerly load the user's services and organizations, and also the service's org and vice versa
-            # (so we can see if the user knows about it)
-            joinedload(User.services).joinedload(Service.organization),
-            joinedload(User.organizations).subqueryload(Organization.services),
-        )
-        .one()
-    )
+    return User.query.filter(
+        User.id == user_id
+    ).options(
+        # eagerly load the user's services and organizations, and also the service's org and vice versa
+        # (so we can see if the user knows about it)
+        joinedload('services'),
+        joinedload('organizations'),
+        joinedload('organizations.services'),
+        joinedload('services.organization'),
+    ).one()
 
 
 @autocommit
@@ -214,13 +159,13 @@ def dao_archive_user(user):
 
     user.organizations = []
 
-    user.auth_type = AuthType.EMAIL
+    user.auth_type = EMAIL_AUTH_TYPE
     user.email_address = get_archived_db_column_value(user.email_address)
     user.mobile_number = None
     user.password = str(uuid.uuid4())
     # Changing the current_session_id signs the user out
-    user.current_session_id = "00000000-0000-0000-0000-000000000000"
-    user.state = "inactive"
+    user.current_session_id = '00000000-0000-0000-0000-000000000000'
+    user.state = 'inactive'
 
     db.session.add(user)
 
@@ -229,30 +174,13 @@ def user_can_be_archived(user):
     active_services = [x for x in user.services if x.active]
 
     for service in active_services:
-        other_active_users = [
-            x for x in service.users if x.state == "active" and x != user
-        ]
+        other_active_users = [x for x in service.users if x.state == 'active' and x != user]
 
         if not other_active_users:
             return False
 
-        if not any(
-            PermissionType.MANAGE_SETTINGS in user.get_permissions(service.id)
-            for user in other_active_users
-        ):
+        if not any('manage_settings' in user.get_permissions(service.id) for user in other_active_users):
             # no-one else has manage settings
             return False
 
     return True
-
-
-def dao_report_users():
-    sql = """
-    select users.name, users.email_address, users.mobile_number, services.name as service_name
-    from users
-    inner join user_to_service on users.id=user_to_service.user_id
-    inner join services on services.id=user_to_service.service_id
-    where services.name not like '_archived%'
-    order by services.name asc, users.name asc
-    """
-    return db.session.execute(text(sql))

app/dao/webauthn_credential_dao.py

@@ -6,7 +6,7 @@ from app.models import WebauthnCredential
 def dao_get_webauthn_credential_by_user_and_id(user_id, webauthn_credential_id):
     return WebauthnCredential.query.filter(
         WebauthnCredential.user_id == user_id,
-        WebauthnCredential.id == webauthn_credential_id,
+        WebauthnCredential.id == webauthn_credential_id
     ).one()
 
 
@@ -22,7 +22,7 @@ def dao_create_webauthn_credential(
         user_id=user_id,
         name=name,
         credential_data=credential_data,
-        registration_response=registration_response,
+        registration_response=registration_response
     )
     db.session.add(webauthn_credential)
     return webauthn_credential

app/delivery/send_to_providers.py

@@ -1,61 +1,55 @@
-import json
+import random
+from datetime import datetime, timedelta
 from urllib import parse
 
 from cachetools import TTLCache, cached
 from flask import current_app
-
-from app import create_uuid, db, notification_provider_clients, redis_store
-from app.aws.s3 import get_personalisation_from_s3, get_phone_number_from_s3
-from app.celery.test_key_tasks import send_email_response, send_sms_response
-from app.dao.email_branding_dao import dao_get_email_branding_by_id
-from app.dao.notifications_dao import dao_update_notification
-from app.dao.provider_details_dao import get_provider_details_by_notification_type
-from app.enums import BrandType, KeyType, NotificationStatus, NotificationType
-from app.exceptions import NotificationTechnicalFailureException
-from app.serialised_models import SerialisedService, SerialisedTemplate
-from app.utils import hilite, utc_now
 from notifications_utils.template import (
     HTMLEmailTemplate,
     PlainTextEmailTemplate,
     SMSMessageTemplate,
 )
 
+from app import create_uuid, db, notification_provider_clients
+from app.celery.research_mode_tasks import (
+    send_email_response,
+    send_sms_response,
+)
+from app.dao.email_branding_dao import dao_get_email_branding_by_id
+from app.dao.notifications_dao import dao_update_notification
+from app.dao.provider_details_dao import (
+    dao_reduce_sms_provider_priority,
+    get_provider_details_by_notification_type,
+)
+from app.exceptions import NotificationTechnicalFailureException
+from app.models import (
+    BRANDING_BOTH,
+    BRANDING_ORG_BANNER,
+    EMAIL_TYPE,
+    KEY_TYPE_TEST,
+    NOTIFICATION_SENDING,
+    NOTIFICATION_STATUS_TYPES_COMPLETED,
+    NOTIFICATION_TECHNICAL_FAILURE,
+    SMS_TYPE,
+)
+from app.serialised_models import SerialisedService, SerialisedTemplate
+
 
 def send_sms_to_provider(notification):
-    """Final step in the message send flow.
-
-    Get data for recipient, template,
-    notification and send it to sns.
-    """
-    # we no longer store the personalisation in the db,
-    # need to retrieve from s3 before generating content
-    # However, we are still sending the initial verify code through personalisation
-    # so if there is some value there, don't overwrite it
-    if not notification.personalisation:
-        personalisation = get_personalisation_from_s3(
-            notification.service_id,
-            notification.job_id,
-            notification.job_row_number,
-        )
-        notification.personalisation = personalisation
-
     service = SerialisedService.from_id(notification.service_id)
     message_id = None
     if not service.active:
         technical_failure(notification=notification)
         return
 
-    if notification.status == NotificationStatus.CREATED:
-        # We get the provider here (which is only aws sns)
-        provider = provider_to_use(NotificationType.SMS, notification.international)
+    if notification.status == 'created':
+        provider = provider_to_use(SMS_TYPE, notification.international)
         if not provider:
             technical_failure(notification=notification)
             return
 
         template_model = SerialisedTemplate.from_id_and_service_id(
-            template_id=notification.template_id,
-            service_id=service.id,
-            version=notification.template_version,
+            template_id=notification.template_id, service_id=service.id, version=notification.template_version
         )
 
         template = SMSMessageTemplate(
@@ -64,7 +58,7 @@ def send_sms_to_provider(notification):
             prefix=service.name,
             show_prefix=service.prefix_sms,
         )
-        if notification.key_type == KeyType.TEST:
+        if service.research_mode or notification.key_type == KEY_TYPE_TEST:
             update_notification_to_sending(notification, provider)
             send_sms_response(provider.name, str(notification.id))
 
@@ -76,118 +70,73 @@ def send_sms_to_provider(notification):
                 # providers as a slow down of our providers can cause us to run out of DB connections
                 # Therefore we pull all the data from our DB models into `send_sms_kwargs`now before
                 # closing the session (as otherwise it would be reopened immediately)
-
-                # We start by trying to get the phone number from a job in s3.  If we fail, we assume
-                # the phone number is for the verification code on login, which is not a job.
-                recipient = None
-                try:
-                    recipient = get_phone_number_from_s3(
-                        notification.service_id,
-                        notification.job_id,
-                        notification.job_row_number,
-                    )
-                except Exception:
-                    # It is our 2facode, maybe
-                    key = f"2facode-{notification.id}".replace(" ", "")
-                    recipient = redis_store.get(key)
-
-                    if recipient:
-                        recipient = recipient.decode("utf-8")
-
-                if recipient is None:
-                    si = notification.service_id
-                    ji = notification.job_id
-                    jrn = notification.job_row_number
-                    raise Exception(
-                        f"The recipient for (Service ID: {si}; Job ID: {ji}; Job Row Number {jrn} was not found."
-                    )
                 send_sms_kwargs = {
-                    "to": recipient,
-                    "content": str(template),
-                    "reference": str(notification.id),
-                    "sender": notification.reply_to_text,
-                    "international": notification.international,
+                    'to': notification.normalised_to,
+                    'content': str(template),
+                    'reference': str(notification.id),
+                    'sender': notification.reply_to_text,
+                    'international': notification.international,
                 }
                 db.session.close()  # no commit needed as no changes to objects have been made above
-
                 message_id = provider.send_sms(**send_sms_kwargs)
-                current_app.logger.info(f"got message_id {message_id}")
             except Exception as e:
-                n = notification
-                msg = f"FAILED send to sms, job_id: {n.job_id} row_number {n.job_row_number} message_id {message_id}"
-                current_app.logger.error(hilite(f"{msg} {e}"))
-
                 notification.billable_units = template.fragment_count
                 dao_update_notification(notification)
+                dao_reduce_sms_provider_priority(provider.name, time_threshold=timedelta(minutes=1))
                 raise e
             else:
-                # Here we map the job_id and row number to the aws message_id
-                n = notification
-                msg = f"Send to aws for job_id {n.job_id} row_number {n.job_row_number} message_id {message_id}"
-                current_app.logger.info(hilite(msg))
                 notification.billable_units = template.fragment_count
                 update_notification_to_sending(notification, provider)
     return message_id
 
 
 def send_email_to_provider(notification):
-    # Someone needs an email, possibly new registration
-    recipient = redis_store.get(f"email-address-{notification.id}")
-    recipient = recipient.decode("utf-8")
-    personalisation = redis_store.get(f"email-personalisation-{notification.id}")
-    if personalisation:
-        personalisation = personalisation.decode("utf-8")
-        notification.personalisation = json.loads(personalisation)
-
     service = SerialisedService.from_id(notification.service_id)
+
     if not service.active:
         technical_failure(notification=notification)
         return
-    if notification.status == NotificationStatus.CREATED:
-        provider = provider_to_use(NotificationType.EMAIL, False)
+    if notification.status == 'created':
+        provider = provider_to_use(EMAIL_TYPE, False)
         template_dict = SerialisedTemplate.from_id_and_service_id(
-            template_id=notification.template_id,
-            service_id=service.id,
-            version=notification.template_version,
+            template_id=notification.template_id, service_id=service.id, version=notification.template_version
         ).__dict__
 
         html_email = HTMLEmailTemplate(
             template_dict,
             values=notification.personalisation,
-            **get_html_email_options(service),
+            **get_html_email_options(service)
         )
 
         plain_text_email = PlainTextEmailTemplate(
-            template_dict, values=notification.personalisation
+            template_dict,
+            values=notification.personalisation
         )
-
-        if notification.key_type == KeyType.TEST:
+        if service.research_mode or notification.key_type == KEY_TYPE_TEST:
             notification.reference = str(create_uuid())
             update_notification_to_sending(notification, provider)
-            send_email_response(notification.reference, recipient)
+            send_email_response(notification.reference, notification.to)
         else:
-            from_address = (
-                f'"{service.name}" <{service.email_from}@'
-                f'{current_app.config["NOTIFY_EMAIL_DOMAIN"]}>'
-            )
+            from_address = '"{}" <{}@{}>'.format(service.name, service.email_from,
+                                                 current_app.config['NOTIFY_EMAIL_DOMAIN'])
 
             reference = provider.send_email(
                 from_address,
-                recipient,
+                notification.normalised_to,
                 plain_text_email.subject,
                 body=str(plain_text_email),
                 html_body=str(html_email),
-                reply_to_address=notification.reply_to_text,
+                reply_to_address=notification.reply_to_text
             )
             notification.reference = reference
             update_notification_to_sending(notification, provider)
 
 
 def update_notification_to_sending(notification, provider):
-    notification.sent_at = utc_now()
+    notification.sent_at = datetime.utcnow()
     notification.sent_by = provider.name
-    if notification.status not in NotificationStatus.completed_types():
-        notification.status = NotificationStatus.SENDING
+    if notification.status not in NOTIFICATION_STATUS_TYPES_COMPLETED:
+        notification.status = NOTIFICATION_SENDING
 
     dao_update_notification(notification)
 
@@ -198,42 +147,43 @@ provider_cache = TTLCache(maxsize=8, ttl=10)
 @cached(cache=provider_cache)
 def provider_to_use(notification_type, international=True):
     active_providers = [
-        p
-        for p in get_provider_details_by_notification_type(
+        p for p in get_provider_details_by_notification_type(
             notification_type, international
-        )
-        if p.active
+        ) if p.active
     ]
 
     if not active_providers:
-        current_app.logger.error(f"{notification_type} failed as no active providers")
-        raise Exception(f"No active {notification_type} providers")
+        current_app.logger.error(
+            "{} failed as no active providers".format(notification_type)
+        )
+        raise Exception("No active {} providers".format(notification_type))
 
-    # we only have sns
-    chosen_provider = active_providers[0]
+    if len(active_providers) == 1:
+        chosen_provider = active_providers[0]
+    else:
+        weights = [p.priority for p in active_providers]
+        chosen_provider = random.choices(active_providers, weights=weights)[0]  # nosec B311 - not sec/crypto related
 
-    return notification_provider_clients.get_client_by_name_and_type(
-        chosen_provider.identifier, notification_type
-    )
+    return notification_provider_clients.get_client_by_name_and_type(chosen_provider.identifier, notification_type)
 
 
 def get_logo_url(base_url, logo_file):
     base_url = parse.urlparse(base_url)
     netloc = base_url.netloc
 
-    if base_url.netloc.startswith("localhost"):
-        netloc = "notify.tools"
-    elif base_url.netloc.startswith("www"):
+    if base_url.netloc.startswith('localhost'):
+        netloc = 'notify.tools'
+    elif base_url.netloc.startswith('www'):
         # strip "www."
         netloc = base_url.netloc[4:]
 
     logo_url = parse.ParseResult(
         scheme=base_url.scheme,
-        netloc="static-logos." + netloc,
+        netloc='static-logos.' + netloc,
         path=logo_file,
         params=base_url.params,
         query=base_url.query,
-        fragment=base_url.fragment,
+        fragment=base_url.fragment
     )
     return parse.urlunparse(logo_url)
 
@@ -241,34 +191,34 @@ def get_logo_url(base_url, logo_file):
 def get_html_email_options(service):
     if service.email_branding is None:
         return {
-            "govuk_banner": True,
-            "brand_banner": False,
+            'govuk_banner': True,
+            'brand_banner': False,
         }
     if isinstance(service, SerialisedService):
         branding = dao_get_email_branding_by_id(service.email_branding)
     else:
         branding = service.email_branding
 
-    logo_url = (
-        get_logo_url(current_app.config["ADMIN_BASE_URL"], branding.logo)
-        if branding.logo
-        else None
-    )
+    logo_url = get_logo_url(
+        current_app.config['ADMIN_BASE_URL'],
+        branding.logo
+    ) if branding.logo else None
 
     return {
-        "govuk_banner": branding.brand_type == BrandType.BOTH,
-        "brand_banner": branding.brand_type == BrandType.ORG_BANNER,
-        "brand_colour": branding.colour,
-        "brand_logo": logo_url,
-        "brand_text": branding.text,
-        "brand_name": branding.name,
+        'govuk_banner': branding.brand_type == BRANDING_BOTH,
+        'brand_banner': branding.brand_type == BRANDING_ORG_BANNER,
+        'brand_colour': branding.colour,
+        'brand_logo': logo_url,
+        'brand_text': branding.text,
+        'brand_name': branding.name,
     }
 
 
 def technical_failure(notification):
-    notification.status = NotificationStatus.TECHNICAL_FAILURE
+    notification.status = NOTIFICATION_TECHNICAL_FAILURE
     dao_update_notification(notification)
     raise NotificationTechnicalFailureException(
-        f"Send {notification.notification_type} for notification id {notification.id} "
-        f"to provider is not allowed: service {notification.service_id} is inactive"
-    )
+        "Send {} for notification id {} to provider is not allowed: service {} is inactive".format(
+            notification.notification_type,
+            notification.id,
+            notification.service_id))

app/docs/__init__.py

@@ -2,10 +2,10 @@ from os import path
 
 from flask import Blueprint, current_app, send_file
 
-docs = Blueprint("docs", __name__, url_prefix="/docs")
+docs = Blueprint('docs', __name__, url_prefix='/docs')
 
 
-@docs.route("/openapi.yml", methods=["GET"])
+@docs.route('/openapi.yml', methods=['GET'])
 def send_openapi():
-    openapi_schema = path.join(current_app.root_path, "../docs/openapi.yml")
-    return send_file(openapi_schema, mimetype="text/yaml"), 200
+    openapi_schema = path.join(current_app.root_path, '../docs/openapi.yml')
+    return send_file(openapi_schema, mimetype='text/yaml'), 200

app/email_branding/email_branding_schema.py

@@ -1,4 +1,4 @@
-from app.enums import BrandType
+from app.models import BRANDING_TYPES
 
 post_create_email_branding_schema = {
     "$schema": "http://json-schema.org/draft-07/schema#",
@@ -9,9 +9,9 @@ post_create_email_branding_schema = {
         "name": {"type": "string"},
         "text": {"type": ["string", "null"]},
         "logo": {"type": ["string", "null"]},
-        "brand_type": {"enum": list(BrandType)},
+        "brand_type": {"enum": BRANDING_TYPES},
     },
-    "required": ["name"],
+    "required": ["name"]
 }
 
 post_update_email_branding_schema = {
@@ -23,7 +23,7 @@ post_update_email_branding_schema = {
         "name": {"type": ["string", "null"]},
         "text": {"type": ["string", "null"]},
         "logo": {"type": ["string", "null"]},
-        "brand_type": {"enum": list(BrandType)},
+        "brand_type": {"enum": BRANDING_TYPES},
     },
-    "required": [],
+    "required": []
 }

app/email_branding/rest.py

@@ -14,45 +14,45 @@ from app.errors import register_errors
 from app.models import EmailBranding
 from app.schema_validation import validate
 
-email_branding_blueprint = Blueprint("email_branding", __name__)
+email_branding_blueprint = Blueprint('email_branding', __name__)
 register_errors(email_branding_blueprint)
 
 
-@email_branding_blueprint.route("", methods=["GET"])
+@email_branding_blueprint.route('', methods=['GET'])
 def get_email_branding_options():
     email_branding_options = [o.serialize() for o in dao_get_email_branding_options()]
     return jsonify(email_branding=email_branding_options)
 
 
-@email_branding_blueprint.route("/<uuid:email_branding_id>", methods=["GET"])
+@email_branding_blueprint.route('/<uuid:email_branding_id>', methods=['GET'])
 def get_email_branding_by_id(email_branding_id):
     email_branding = dao_get_email_branding_by_id(email_branding_id)
     return jsonify(email_branding=email_branding.serialize())
 
 
-@email_branding_blueprint.route("", methods=["POST"])
+@email_branding_blueprint.route('', methods=['POST'])
 def create_email_branding():
     data = request.get_json()
 
     validate(data, post_create_email_branding_schema)
 
     email_branding = EmailBranding(**data)
-    if "text" not in data.keys():
+    if 'text' not in data.keys():
         email_branding.text = email_branding.name
 
     dao_create_email_branding(email_branding)
     return jsonify(data=email_branding.serialize()), 201
 
 
-@email_branding_blueprint.route("/<uuid:email_branding_id>", methods=["POST"])
+@email_branding_blueprint.route('/<uuid:email_branding_id>', methods=['POST'])
 def update_email_branding(email_branding_id):
     data = request.get_json()
 
     validate(data, post_update_email_branding_schema)
 
     fetched_email_branding = dao_get_email_branding_by_id(email_branding_id)
-    if "text" not in data.keys() and "name" in data.keys():
-        data["text"] = data["name"]
+    if 'text' not in data.keys() and 'name' in data.keys():
+        data['text'] = data['name']
     dao_update_email_branding(fetched_email_branding, **data)
 
     return jsonify(data=fetched_email_branding.serialize()), 200

app/enums.py

@@ -1,213 +0,0 @@
-from enum import StrEnum
-
-
-class TemplateType(StrEnum):
-    SMS = "sms"
-    EMAIL = "email"
-    LETTER = "letter"
-
-
-class NotificationType(StrEnum):
-    SMS = "sms"
-    EMAIL = "email"
-    LETTER = "letter"
-
-
-class TemplateProcessType(StrEnum):
-    # TODO: Should Template.process_type be changed to use this?
-    NORMAL = "normal"
-    PRIORITY = "priority"
-
-
-class AuthType(StrEnum):
-    SMS = "sms_auth"
-    EMAIL = "email_auth"
-    WEBAUTHN = "webauthn_auth"
-
-
-class CallbackType(StrEnum):
-    DELIVERY_STATUS = "delivery_status"
-    COMPLAINT = "complaint"
-
-
-class OrganizationType(StrEnum):
-    FEDERAL = "federal"
-    STATE = "state"
-    OTHER = "other"
-
-
-class NotificationStatus(StrEnum):
-    CANCELLED = "cancelled"
-    CREATED = "created"
-    SENDING = "sending"
-    SENT = "sent"
-    DELIVERED = "delivered"
-    PENDING = "pending"
-    FAILED = "failed"
-    TECHNICAL_FAILURE = "technical-failure"
-    TEMPORARY_FAILURE = "temporary-failure"
-    PERMANENT_FAILURE = "permanent-failure"
-    PENDING_VIRUS_CHECK = "pending-virus-check"
-    VALIDATION_FAILED = "validation-failed"
-    VIRUS_SCAN_FAILED = "virus-scan-failed"
-
-    @classmethod
-    def failed_types(cls) -> tuple[str, ...]:
-        return (
-            cls.TECHNICAL_FAILURE,
-            cls.TEMPORARY_FAILURE,
-            cls.PERMANENT_FAILURE,
-            cls.VALIDATION_FAILED,
-            cls.VIRUS_SCAN_FAILED,
-        )
-
-    @classmethod
-    def completed_types(cls) -> tuple[str, ...]:
-        return (
-            cls.SENT,
-            cls.DELIVERED,
-            cls.FAILED,
-            cls.TECHNICAL_FAILURE,
-            cls.TEMPORARY_FAILURE,
-            cls.PERMANENT_FAILURE,
-            cls.CANCELLED,
-        )
-
-    @classmethod
-    def success_types(cls) -> tuple[str, ...]:
-        return (cls.SENT, cls.DELIVERED)
-
-    @classmethod
-    def billable_types(cls) -> tuple[str, ...]:
-        return (
-            cls.SENDING,
-            cls.SENT,
-            cls.DELIVERED,
-            cls.PENDING,
-            cls.FAILED,
-            cls.TEMPORARY_FAILURE,
-            cls.PERMANENT_FAILURE,
-        )
-
-    @classmethod
-    def billable_sms_types(cls) -> tuple[str, ...]:
-        return (
-            cls.SENDING,
-            cls.SENT,  # internationally
-            cls.DELIVERED,
-            cls.PENDING,
-            cls.TEMPORARY_FAILURE,
-            cls.PERMANENT_FAILURE,
-        )
-
-    @classmethod
-    def sent_email_types(cls) -> tuple[str, ...]:
-        return (
-            cls.SENDING,
-            cls.DELIVERED,
-            cls.TEMPORARY_FAILURE,
-            cls.PERMANENT_FAILURE,
-        )
-
-    @classmethod
-    def non_billable_types(cls) -> tuple[str, ...]:
-        return tuple(set(cls) - set(cls.billable_types()))
-
-
-class PermissionType(StrEnum):
-    MANAGE_USERS = "manage_users"
-    MANAGE_TEMPLATES = "manage_templates"
-    MANAGE_SETTINGS = "manage_settings"
-    SEND_TEXTS = "send_texts"
-    SEND_EMAILS = "send_emails"
-    MANAGE_API_KEYS = "manage_api_keys"
-    PLATFORM_ADMIN = "platform_admin"
-    VIEW_ACTIVITY = "view_activity"
-
-    @classmethod
-    def defaults(cls) -> tuple[str, ...]:
-        return (
-            cls.MANAGE_USERS,
-            cls.MANAGE_TEMPLATES,
-            cls.MANAGE_SETTINGS,
-            cls.SEND_TEXTS,
-            cls.SEND_EMAILS,
-            cls.MANAGE_API_KEYS,
-            cls.VIEW_ACTIVITY,
-        )
-
-
-class ServicePermissionType(StrEnum):
-    EMAIL = "email"
-    SMS = "sms"
-    INTERNATIONAL_SMS = "international_sms"
-    INBOUND_SMS = "inbound_sms"
-    SCHEDULE_NOTIFICATIONS = "schedule_notifications"
-    EMAIL_AUTH = "email_auth"
-    UPLOAD_DOCUMENT = "upload_document"
-    EDIT_FOLDER_PERMISSIONS = "edit_folder_permissions"
-
-    @classmethod
-    def defaults(cls) -> tuple[str, ...]:
-        return (
-            cls.SMS,
-            cls.EMAIL,
-            cls.INTERNATIONAL_SMS,
-        )
-
-
-class RecipientType(StrEnum):
-    MOBILE = "mobile"
-    EMAIL = "email"
-
-
-class KeyType(StrEnum):
-    NORMAL = "normal"
-    TEAM = "team"
-    TEST = "test"
-
-
-class JobStatus(StrEnum):
-    PENDING = "pending"
-    IN_PROGRESS = "in progress"
-    FINISHED = "finished"
-    SENDING_LIMITS_EXCEEDED = "sending limits exceeded"
-    SCHEDULED = "scheduled"
-    CANCELLED = "cancelled"
-    READY_TO_SEND = "ready to send"
-    SENT_TO_DVLA = "sent to dvla"
-    ERROR = "error"
-
-
-class InvitedUserStatus(StrEnum):
-    PENDING = "pending"
-    ACCEPTED = "accepted"
-    CANCELLED = "cancelled"
-    EXPIRED = "expired"
-
-
-class BrandType(StrEnum):
-    ORG = "org"
-    BOTH = "both"
-    ORG_BANNER = "org_banner"
-
-
-class CodeType(StrEnum):
-    EMAIL = "email"
-    SMS = "sms"
-
-
-class AgreementType(StrEnum):
-    MOU = "MOU"
-    IAA = "IAA"
-
-
-class AgreementStatus(StrEnum):
-    ACTIVE = "active"
-    EXPIRED = "expired"
-
-
-class StatisticsType(StrEnum):
-    REQUESTED = "requested"
-    DELIVERED = "delivered"
-    FAILURE = "failure"

app/errors.py

@@ -1,17 +1,17 @@
 from flask import current_app, json, jsonify
 from jsonschema import ValidationError as JsonSchemaValidationError
 from marshmallow import ValidationError
+from notifications_utils.recipients import InvalidEmailError
 from sqlalchemy.exc import DataError
 from sqlalchemy.orm.exc import NoResultFound
 
 from app.authentication.auth import AuthError
-from app.enums import KeyType
 from app.exceptions import ArchiveValidationError
-from notifications_utils.recipients import InvalidEmailError
 
 
 class VirusScanError(Exception):
     def __init__(self, message):
+
         super().__init__(message)
 
 
@@ -25,37 +25,41 @@ class InvalidRequest(Exception):
         self.status_code = status_code
 
     def to_dict(self):
-        return {"result": "error", "message": self.message}
+        return {'result': 'error', 'message': self.message}
 
     def to_dict_v2(self):
-        """
+        '''
         Version 2 of the public api error response.
-        """
+        '''
         return {
             "status_code": self.status_code,
-            "errors": [{"error": self.__class__.__name__, "message": self.message}],
+            "errors": [
+                {
+                    "error": self.__class__.__name__,
+                    "message": self.message
+                }
+            ]
         }
 
     def __str__(self):
         return str(self.to_dict())
 
 
-# TODO maintainability what is this for?  How to unit test it?
 def register_errors(blueprint):
     @blueprint.errorhandler(InvalidEmailError)
     def invalid_format(error):
         # Please not that InvalidEmailError is re-raised for InvalidEmail or InvalidPhone,
         # work should be done in the utils app to tidy up these errors.
-        return jsonify(result="error", message=str(error)), 400
+        return jsonify(result='error', message=str(error)), 400
 
     @blueprint.errorhandler(AuthError)
     def authentication_error(error):
-        return jsonify(result="error", message=error.message), error.code
+        return jsonify(result='error', message=error.message), error.code
 
     @blueprint.errorhandler(ValidationError)
     def marshmallow_validation_error(error):
         current_app.logger.info(error)
-        return jsonify(result="error", message=error.messages), 400
+        return jsonify(result='error', message=error.messages), 400
 
     @blueprint.errorhandler(JsonSchemaValidationError)
     def jsonschema_validation_error(error):
@@ -65,7 +69,7 @@ def register_errors(blueprint):
     @blueprint.errorhandler(ArchiveValidationError)
     def archive_validation_error(error):
         current_app.logger.info(error)
-        return jsonify(result="error", message=str(error)), 400
+        return jsonify(result='error', message=str(error)), 400
 
     @blueprint.errorhandler(InvalidRequest)
     def invalid_data(error):
@@ -78,32 +82,28 @@ def register_errors(blueprint):
     def bad_request(e):
         msg = e.description or "Invalid request parameters"
         current_app.logger.exception(msg)
-        return jsonify(result="error", message=str(msg)), 400
+        return jsonify(result='error', message=str(msg)), 400
 
     @blueprint.errorhandler(401)
     def unauthorized(e):
         error_message = "Unauthorized: authentication token must be provided"
-        return (
-            jsonify(result="error", message=error_message),
-            401,
-            [("WWW-Authenticate", "Bearer")],
-        )
+        return jsonify(result='error', message=error_message), 401, [('WWW-Authenticate', 'Bearer')]
 
     @blueprint.errorhandler(403)
     def forbidden(e):
         error_message = "Forbidden: invalid authentication token provided"
-        return jsonify(result="error", message=error_message), 403
+        return jsonify(result='error', message=error_message), 403
 
     @blueprint.errorhandler(429)
     def limit_exceeded(e):
         current_app.logger.exception(e)
-        return jsonify(result="error", message=str(e.description)), 429
+        return jsonify(result='error', message=str(e.description)), 429
 
     @blueprint.errorhandler(NoResultFound)
     @blueprint.errorhandler(DataError)
     def no_result_found(e):
         current_app.logger.info(e)
-        return jsonify(result="error", message="No result found"), 404
+        return jsonify(result='error', message="No result found"), 404
 
     # this must be defined after all other error handlers since it catches the generic Exception object
     @blueprint.app_errorhandler(500)
@@ -111,48 +111,6 @@ def register_errors(blueprint):
     def internal_server_error(e):
         # if e is a werkzeug InternalServerError then it may wrap the original exception. For more details see:
         # https://flask.palletsprojects.com/en/1.1.x/errorhandling/?highlight=internalservererror#unhandled-exceptions
-        e = getattr(e, "original_exception", e)
+        e = getattr(e, 'original_exception', e)
         current_app.logger.exception(e)
-        return jsonify(result="error", message="Internal server error"), 500
-
-
-class TooManyRequestsError(InvalidRequest):
-    status_code = 429
-    message_template = "Exceeded send limits ({}) for today"
-
-    def __init__(self, sending_limit):
-        self.message = self.message_template.format(sending_limit)
-
-
-class TotalRequestsError(InvalidRequest):
-    status_code = 429
-    message_template = "Exceeded total application limits ({}) for today"
-
-    def __init__(self, sending_limit):
-        self.message = self.message_template.format(sending_limit)
-
-
-class RateLimitError(InvalidRequest):
-    status_code = 429
-    message_template = (
-        "Exceeded rate limit for key type {} of {} requests per {} seconds"
-    )
-
-    def __init__(self, sending_limit, interval, key_type):
-        # normal keys are spoken of as "live" in the documentation
-        # so using this in the error messaging
-        if key_type == KeyType.NORMAL:
-            key_type = "live"
-
-        self.message = self.message_template.format(
-            key_type.upper(), sending_limit, interval
-        )
-
-
-class BadRequestError(InvalidRequest):
-    message = "An error occurred"
-
-    def __init__(self, fields=None, message=None, status_code=400):
-        self.status_code = status_code
-        self.fields = fields or []
-        self.message = message if message else self.message
+        return jsonify(result='error', message="Internal server error"), 500

app/events/rest.py

@@ -4,11 +4,11 @@ from app.dao.events_dao import dao_create_event
 from app.errors import register_errors
 from app.schemas import event_schema
 
-events = Blueprint("events", __name__, url_prefix="/events")
+events = Blueprint('events', __name__, url_prefix='/events')
 register_errors(events)
 
 
-@events.route("", methods=["POST"])
+@events.route('', methods=['POST'])
 def create_event():
     data = request.get_json()
     event = event_schema.load(data)

app/hashing.py

@@ -2,7 +2,7 @@ from flask_bcrypt import check_password_hash, generate_password_hash
 
 
 def hashpw(password):
-    return generate_password_hash(password.encode("UTF-8"), 10).decode("utf-8")
+    return generate_password_hash(password.encode('UTF-8'), 10).decode('utf-8')
 
 
 def check_hash(password, hashed_password):

app/history_meta.py

@@ -14,13 +14,13 @@ Lastly when to create a version is done manually in dao_utils version decorator
 session events.
 
 """
+import datetime
 
 from sqlalchemy import Column, ForeignKeyConstraint, Integer, Table, util
-from sqlalchemy.orm import attributes, object_mapper, registry
+from sqlalchemy.ext.declarative import declared_attr
+from sqlalchemy.orm import attributes, mapper, object_mapper
 from sqlalchemy.orm.properties import ColumnProperty, RelationshipProperty
 
-from app.utils import utc_now
-
 
 def col_references_table(col, table):
     for fk in col.foreign_keys:
@@ -43,13 +43,27 @@ def _history_mapper(local_mapper):  # noqa (C901 too complex)
         getattr(local_mapper.class_, prop.key).impl.active_history = True
 
     super_mapper = local_mapper.inherits
-    super_history_mapper = getattr(cls, "__history_mapper__", None)
+    super_history_mapper = getattr(cls, '__history_mapper__', None)
 
     polymorphic_on = None
     super_fks = []
 
+    def _col_copy(col):
+        orig = col
+        col = col.copy()
+        orig.info['history_copy'] = col
+        col.unique = False
+
+        # if the column is nullable, we could end up overwriting an on-purpose null value with a default.
+        # if it's not nullable, however, the default may be relied upon to correctly set values within the database,
+        # so we should preserve it
+        if col.nullable:
+            col.default = col.server_default = None
+        return col
+
     properties = util.OrderedDict()
-    if not super_mapper or local_mapper.local_table is not super_mapper.local_table:
+    if not super_mapper or \
+            local_mapper.local_table is not super_mapper.local_table:
         cols = []
         version_meta = {"version_meta": True}
         for column in local_mapper.local_table.c:
@@ -57,9 +71,15 @@ def _history_mapper(local_mapper):  # noqa (C901 too complex)
                 continue
 
             col = _col_copy(column)
-            _add_primary_keys_to_super_fks(
-                super_mapper, column, super_fks, super_history_mapper, col
-            )
+
+            if super_mapper and \
+                    col_references_table(column, super_mapper.local_table):
+                super_fks.append(
+                    (
+                        col.key,
+                        list(super_history_mapper.local_table.primary_key)[0]
+                    )
+                )
 
             cols.append(col)
 
@@ -68,132 +88,82 @@ def _history_mapper(local_mapper):  # noqa (C901 too complex)
 
             orig_prop = local_mapper.get_property_by_column(column)
             # carry over column re-mappings
-            if len(orig_prop.columns) > 1 or orig_prop.columns[0].key != orig_prop.key:
+            if len(orig_prop.columns) > 1 or \
+                    orig_prop.columns[0].key != orig_prop.key:
                 properties[orig_prop.key] = tuple(
-                    col.info["history_copy"] for col in orig_prop.columns
-                )
+                    col.info['history_copy'] for col in orig_prop.columns)
 
-        _add_version_to_super_fks(super_fks, super_mapper, super_history_mapper)
+        if super_mapper:
+            super_fks.append(
+                (
+                    'version', super_history_mapper.local_table.c.version
+                )
+            )
 
         # "version" stores the integer version id.  This column is
         # required.
         cols.append(
             Column(
-                "version",
-                Integer,
-                primary_key=True,
-                autoincrement=False,
-                info=version_meta,
-            )
-        )
+                'version', Integer, primary_key=True,
+                autoincrement=False, info=version_meta))
 
-        _handle_super_fks(super_fks, cols)
+        if super_fks:
+            cols.append(ForeignKeyConstraint(*zip(*super_fks)))
 
         table = Table(
-            local_mapper.local_table.name + "_history",
+            local_mapper.local_table.name + '_history',
             local_mapper.local_table.metadata,
             *cols,
             schema=local_mapper.local_table.schema
         )
     else:
-        table = _handle_single_table_inheritance(local_mapper, super_history_mapper)
+        # single table inheritance.  take any additional columns that may have
+        # been added and add them to the history table.
+        for column in local_mapper.local_table.c:
+            if column.key not in super_history_mapper.local_table.c:
+                col = _col_copy(column)
+                super_history_mapper.local_table.append_column(col)
+        table = None
 
-    bases = _get_bases_for_versioned_class(
-        super_history_mapper, table, properties, local_mapper
-    )
-    versioned_cls = type.__new__(type, "%sHistory" % cls.__name__, bases, {})
-
-    m = mapper_registry.map_imperatively(
-        versioned_cls,
-        table,
-        with_polymorphic=("*", super_history_mapper),
-        polymorphic_on=polymorphic_on,
-        polymorphic_identity=local_mapper.polymorphic_identity,
-        properties=properties,
-    )
-    cls.__history_mapper__ = m
-    _add_version_for_non_super_history_mapper(super_history_mapper, local_mapper)
-
-
-def _add_primary_keys_to_super_fks(
-    super_mapper, column, super_fks, super_history_mapper, col
-):
-    if super_mapper and col_references_table(column, super_mapper.local_table):
-        super_fks.append(
-            (col.key, list(super_history_mapper.local_table.primary_key)[0])
-        )
-
-
-def _add_version_to_super_fks(super_fks, super_mapper, super_history_mapper):
-    if super_mapper:
-        super_fks.append(("version", super_history_mapper.local_table.c.version))
-
-
-def _handle_super_fks(super_fks, cols):
-    if super_fks:
-        cols.append(ForeignKeyConstraint(*zip(*super_fks)))
-
-
-def _handle_single_table_inheritance(local_mapper, super_history_mapper):
-    # single table inheritance.  take any additional columns that may have
-    # been added and add them to the history table.
-    for column in local_mapper.local_table.c:
-        if column.key not in super_history_mapper.local_table.c:
-            col = _col_copy(column)
-            super_history_mapper.local_table.append_column(col)
-    return None
-
-
-def _get_bases_for_versioned_class(
-    super_history_mapper, table, properties, local_mapper
-):
     if super_history_mapper:
         bases = (super_history_mapper.class_,)
 
         if table is not None:
-            properties["changed"] = (table.c.changed,) + tuple(
-                super_history_mapper.attrs.changed.columns
+            properties['changed'] = (
+                (table.c.changed, ) +
+                tuple(super_history_mapper.attrs.changed.columns)
             )
 
     else:
         bases = local_mapper.base_mapper.class_.__bases__
-    return bases
+    versioned_cls = type.__new__(type, "%sHistory" % cls.__name__, bases, {})
 
+    m = mapper(
+        versioned_cls,
+        table,
+        inherits=super_history_mapper,
+        polymorphic_on=polymorphic_on,
+        polymorphic_identity=local_mapper.polymorphic_identity,
+        properties=properties
+    )
+    cls.__history_mapper__ = m
 
-def _add_version_for_non_super_history_mapper(super_history_mapper, local_mapper):
     if not super_history_mapper:
         local_mapper.local_table.append_column(
-            Column("version", Integer, default=1, nullable=False)
+            Column('version', Integer, default=1, nullable=False)
         )
-        local_mapper.add_property("version", local_mapper.local_table.c.version)
+        local_mapper.add_property(
+            "version", local_mapper.local_table.c.version)
 
 
-def _col_copy(col):
-    orig = col
-    col = Column(
-        col.name, col.type, nullable=col.nullable, unique=False, default=col.default
-    )
-    orig.info["history_copy"] = col
-
-    # if the column is nullable, we could end up overwriting an on-purpose null value with a default.
-    # if it's not nullable, however, the default may be relied upon to correctly set values within the database,
-    # so we should preserve it
-    if col.nullable:
-        col.default = col.server_default = None
-    return col
-
-
-mapper_registry = registry()
-
-
-@mapper_registry.mapped
 class Versioned(object):
-    __abstract__ = True
-
-    @classmethod
-    def __declare_last__(cls):
-        if not hasattr(cls, "__history_mapper__"):
-            _history_mapper(cls.__mapper__)
+    @declared_attr
+    def __mapper_cls__(cls):
+        def map(cls, *arg, **kw):
+            mp = mapper(cls, *arg, **kw)
+            _history_mapper(mp)
+            return mp
+        return map
 
     @classmethod
     def get_history_model(cls):
@@ -211,6 +181,7 @@ def create_history(obj, history_cls=None):
     obj_state = attributes.instance_state(obj)
     data = {}
     for prop in obj_mapper.iterate_properties:
+
         # expired object attributes and also deferred cols might not
         # be in the dict.  force it them load no matter what by using getattr().
         if prop.key not in obj_state.dict:
@@ -229,21 +200,21 @@ def create_history(obj, history_cls=None):
         # not yet have a value before insert
 
         elif isinstance(prop, RelationshipProperty):
-            if hasattr(history_cls, prop.key + "_id"):
+            if hasattr(history_cls, prop.key + '_id'):
                 foreign_obj = getattr(obj, prop.key)
                 # if it's a nullable relationship, foreign_obj will be None, and we actually want to record that
-                data[prop.key + "_id"] = getattr(foreign_obj, "id", None)
+                data[prop.key + '_id'] = getattr(foreign_obj, 'id', None)
 
     if not obj.version:
         obj.version = 1
-        obj.created_at = utc_now()
+        obj.created_at = datetime.datetime.utcnow()
     else:
         obj.version += 1
-        now = utc_now()
+        now = datetime.datetime.utcnow()
         obj.updated_at = now
-        data["updated_at"] = now
+        data['updated_at'] = now
 
-    data["version"] = obj.version
-    data["created_at"] = obj.created_at
+    data['version'] = obj.version
+    data['created_at'] = obj.created_at
 
     return history_cls(**data)

app/inbound_number/rest.py

@@ -8,33 +8,31 @@ from app.dao.inbound_numbers_dao import (
 )
 from app.errors import register_errors
 
-inbound_number_blueprint = Blueprint(
-    "inbound_number", __name__, url_prefix="/inbound-number"
-)
+inbound_number_blueprint = Blueprint('inbound_number', __name__, url_prefix='/inbound-number')
 register_errors(inbound_number_blueprint)
 
 
-@inbound_number_blueprint.route("", methods=["GET"])
+@inbound_number_blueprint.route('', methods=['GET'])
 def get_inbound_numbers():
     inbound_numbers = [i.serialize() for i in dao_get_inbound_numbers()]
 
     return jsonify(data=inbound_numbers if inbound_numbers else [])
 
 
-@inbound_number_blueprint.route("/service/<uuid:service_id>", methods=["GET"])
+@inbound_number_blueprint.route('/service/<uuid:service_id>', methods=['GET'])
 def get_inbound_number_for_service(service_id):
     inbound_number = dao_get_inbound_number_for_service(service_id)
 
     return jsonify(data=inbound_number.serialize() if inbound_number else {})
 
 
-@inbound_number_blueprint.route("/service/<uuid:service_id>/off", methods=["POST"])
+@inbound_number_blueprint.route('/service/<uuid:service_id>/off', methods=['POST'])
 def post_set_inbound_number_off(service_id):
     dao_set_inbound_number_active_flag(service_id, active=False)
     return jsonify(), 204
 
 
-@inbound_number_blueprint.route("/available", methods=["GET"])
+@inbound_number_blueprint.route('/available', methods=['GET'])
 def get_available_inbound_numbers():
     inbound_numbers = [i.serialize() for i in dao_get_available_inbound_numbers()]