latest

remove copilot

.github/actions/setup-project/action.yml

@@ -0,0 +1,15 @@
+name: Set up project
+description: Setup python & install dependencies
+runs:
+  using: composite
+  steps:
+    - name: Install container dependencies
+      shell: bash
+      run: |
+        sudo apt-get update \
+        && sudo apt-get install -y --no-install-recommends \
+        libcurl4-openssl-dev
+    - name: Set up Python 3.9
+      uses: actions/setup-python@v3
+      with:
+        python-version: "3.9"

.github/workflows/checks.yml

@@ -8,7 +8,7 @@ permissions:
 env:
   DEBUG: True
   ANTIVIRUS_ENABLED: 0
-  NOTIFY_ENVIRONMENT: development
+  NOTIFY_ENVIRONMENT: test
   NOTIFICATION_QUEUE_PREFIX: local_dev_10x
   STATSD_HOST: localhost
   SES_STUB_URL: None
@@ -25,8 +25,6 @@ env:
   API_HOST_NAME: http://localhost:6011
   REDIS_URL: redis://localhost:6380
   REDIS_ENABLED: False
-  SQLALCHEMY_DATABASE_URI: postgresql://postgres:chummy@db:5432/notification_api
-  SQLALCHEMY_DATABASE_TEST_URI: postgresql://user:password@localhost:5432/test_notification_api
   AWS_REGION: us-west-2
   AWS_PINPOINT_REGION: us-west-2
   AWS_US_TOLL_FREE_NUMBER: +18446120782
@@ -52,25 +50,75 @@ jobs:
           - 5432:5432
 
     steps:
-    - name: Install container dependencies
-      run: |
-        sudo apt-get update \
-        && sudo apt-get install -y --no-install-recommends \
-        libcurl4-openssl-dev
     - uses: actions/checkout@v3
-    - name: Set up Python 3.9
+    - uses: ./.github/actions/setup-project
-      uses: actions/setup-python@v3
-      with:
-        python-version: "3.9"
     - name: Install application dependencies
       run: make bootstrap
       env:
         SQLALCHEMY_DATABASE_TEST_URI: postgresql://user:password@localhost:5432/test_notification_api
     # - name: Run style checks
     #   run: flake8 .
-    # - name: Check imports alphabetized
+    - name: Check imports alphabetized
-    #   run: isort --check-only ./app ./tests
+      run: isort --check-only ./app ./tests
     - name: Run tests
       run: pytest -n4 --maxfail=10
       env:
         SQLALCHEMY_DATABASE_TEST_URI: postgresql://user:password@localhost:5432/test_notification_api
+
+  pip-audit:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/setup-project
+      - uses: trailofbits/gh-action-pip-audit@v1.0.0
+        with:
+          inputs: requirements.txt requirements_for_test.txt
+          ignore-vulns: PYSEC-2022-237
+
+  static-scan:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/setup-project
+      - name: Install bandit
+        run: pip install bandit
+      - name: Run scan
+        run: bandit -r app/ --confidence-level medium
+
+  dynamic-scan:
+    runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_USER: user
+          POSTGRES_PASSWORD: password
+          POSTGRES_DB: test_notification_api
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          # Maps tcp port 5432 on service container to the host
+          - 5432:5432
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/setup-project
+      - name: Install application dependencies
+        run: make bootstrap
+        env:
+          SQLALCHEMY_DATABASE_TEST_URI: postgresql://user:password@localhost:5432/test_notification_api
+      - name: Run server
+        run: make run-flask &
+        env:
+          SQLALCHEMY_DATABASE_TEST_URI: postgresql://user:password@localhost:5432/test_notification_api
+      - name: Run OWASP Baseline Scan
+        uses: zaproxy/action-api-scan@v0.1.1
+        with:
+          docker_name: 'owasp/zap2docker-weekly'
+          target: 'http://localhost:6011/_status'
+          fail_action: true
+          allow_issue_writing: false
+          rules_file_name: 'zap.conf'
+          cmd_options: '-I'

.github/workflows/daily_checks.yml

@@ -0,0 +1,93 @@
+name: Run daily scans
+
+on:
+  schedule:
+    # cron format: 'minute hour dayofmonth month dayofweek'
+    # this will run at noon UTC every day (7am EST / 8am EDT)
+    - cron: '0 12 * * *'
+
+permissions:
+  contents: read
+
+env:
+  DEBUG: True
+  ANTIVIRUS_ENABLED: 0
+  NOTIFY_ENVIRONMENT: test
+  NOTIFICATION_QUEUE_PREFIX: local_dev_10x
+  STATSD_HOST: localhost
+  SES_STUB_URL: None
+  NOTIFY_APP_NAME: api
+  NOTIFY_EMAIL_DOMAIN: dispostable.com
+  NOTIFY_LOG_PATH: /workspace/logs/app.log
+  ADMIN_CLIENT_ID: notify-admin
+  ADMIN_CLIENT_SECRET: dev-notify-secret-key
+  GOVUK_ALERTS_CLIENT_ID: govuk-alerts
+  FLASK_APP: application.py
+  FLASK_ENV: development
+  WERKZEUG_DEBUG_PIN: off
+  ADMIN_BASE_URL: http://localhost:6012
+  API_HOST_NAME: http://localhost:6011
+  REDIS_URL: redis://localhost:6380
+  REDIS_ENABLED: False
+  AWS_REGION: us-west-2
+  AWS_PINPOINT_REGION: us-west-2
+  AWS_US_TOLL_FREE_NUMBER: +18446120782
+
+jobs:
+  pip-audit:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/setup-project
+      - uses: trailofbits/gh-action-pip-audit@v1.0.0
+        with:
+          inputs: requirements.txt requirements_for_test.txt
+          ignore-vulns: PYSEC-2022-237
+
+  static-scan:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/setup-project
+      - name: Install bandit
+        run: pip install bandit
+      - name: Run scan
+        run: bandit -r app/ --confidence-level medium
+
+  dynamic-scan:
+    runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_USER: user
+          POSTGRES_PASSWORD: password
+          POSTGRES_DB: test_notification_api
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          # Maps tcp port 5432 on service container to the host
+          - 5432:5432
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/setup-project
+      - name: Install application dependencies
+        run: make bootstrap
+        env:
+          SQLALCHEMY_DATABASE_TEST_URI: postgresql://user:password@localhost:5432/test_notification_api
+      - name: Run server
+        run: make run-flask &
+        env:
+          SQLALCHEMY_DATABASE_TEST_URI: postgresql://user:password@localhost:5432/test_notification_api
+      - name: Run OWASP Baseline Scan
+        uses: zaproxy/action-api-scan@v0.1.1
+        with:
+          docker_name: 'owasp/zap2docker-weekly'
+          target: 'http://localhost:6011/_status'
+          fail_action: true
+          allow_issue_writing: false
+          rules_file_name: 'zap.conf'
+          cmd_options: '-I'

.github/workflows/deploy.yml

@@ -35,8 +35,9 @@ jobs:
     - name: Deploy to cloud.gov
       uses: 18f/cg-deploy-action@main
       env:
-        DANGEROUS_SALT: ${{ secrets.DANGEROUS_SALT }}
+        DANGEROUS_SALT: ${{ secrets.PROD_DANGEROUS_SALT }}
-        SECRET_KEY: ${{ secrets.SECRET_KEY }}
+        SECRET_KEY: ${{ secrets.PROD_SECRET_KEY }}
+        ADMIN_CLIENT_SECRET: ${{ secrets.PROD_ADMIN_CLIENT_SECRET }}
         AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
         AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
       with:
@@ -44,11 +45,11 @@ jobs:
         cf_password: ${{ secrets.cloudgov_password }}
         cf_org: gsa-10x-prototyping
         cf_space: 10x-notifications
-        full_command: |
+        push_arguments: >-
-          cf push --strategy rolling \
+          --var DANGEROUS_SALT="$DANGEROUS_SALT"
-          --var DANGEROUS_SALT="$DANGEROUS_SALT" \
+          --var SECRET_KEY="$SECRET_KEY"
-          --var SECRET_KEY="$SECRET_KEY" \
+          --var ADMIN_CLIENT_SECRET="$ADMIN_CLIENT_SECRET"
-          --var AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" \
+          --var AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID"
           --var AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY"
 
   bail:

.gitignore

@@ -4,6 +4,7 @@ queues.csv
 __pycache__/
 *.py[cod]
 
+.venv/
 venv/
 venv-freeze/
 

Makefile

@@ -13,11 +13,6 @@ CF_SPACE ?= ${DEPLOY_ENV}
 CF_HOME ?= ${HOME}
 $(eval export CF_HOME)
 
-CF_MANIFEST_PATH ?= /tmp/manifest.yml
-
-
-NOTIFY_CREDENTIALS ?= ~/.notify-credentials
-
 
 ## DEVELOPMENT
 
@@ -76,6 +71,17 @@ test: ## Run tests
 freeze-requirements: ## Pin all requirements including sub dependencies into requirements.txt
 	pip install --upgrade pip-tools
 	pip-compile requirements.in
+	pip install -r requirements.txt
+
+.PHONY: audit
+audit:
+	pip install --upgrade pip-audit
+	pip-audit -r requirements.txt -r requirements_for_test.txt -l --ignore-vuln PYSEC-2022-237
+
+.PHONY: static-scan
+static-scan:
+	pip install bandit
+	bandit -r app/
 
 .PHONY: clean
 clean:
@@ -110,19 +116,6 @@ cf-login: ## Log in to Cloud Foundry
 	@echo "Logging in to Cloud Foundry on ${CF_API}"
 	@cf login -a "${CF_API}" -u ${CF_USERNAME} -p "${CF_PASSWORD}" -o "${CF_ORG}" -s "${CF_SPACE}"
 
-.PHONY: generate-manifest
-generate-manifest:
-	$(if ${CF_APP},,$(error Must specify CF_APP))
-	$(if ${CF_SPACE},,$(error Must specify CF_SPACE))
-	$(if $(shell which gpg2), $(eval export GPG=gpg2), $(eval export GPG=gpg))
-	$(if ${GPG_PASSPHRASE_TXT}, $(eval export DECRYPT_CMD=echo -n $$$${GPG_PASSPHRASE_TXT} | ${GPG} --quiet --batch --passphrase-fd 0 --pinentry-mode loopback -d), $(eval export DECRYPT_CMD=${GPG} --quiet --batch -d))
-
-	@jinja2 --strict manifest.yml.j2 \
-	    -D environment=${CF_SPACE} \
-	    -D CF_APP=${CF_APP} \
-	    --format=yaml \
-	    <(${DECRYPT_CMD} ${NOTIFY_CREDENTIALS}/credentials/${CF_SPACE}/paas/environment-variables.gpg) 2>&1
-
 .PHONY: cf-deploy
 cf-deploy: ## Deploys the app to Cloud Foundry
 	$(if ${CF_SPACE},,$(error Must specify CF_SPACE))
@@ -145,11 +138,11 @@ cf-deploy-api-db-migration:
 	cf push notifications-api --no-route -f ${CF_MANIFEST_PATH}
 	rm ${CF_MANIFEST_PATH}
 
-	cf run-task notifications-api --command="flask db upgrade" --name api_db_migration
+	cf run-task notify-api-alt --command="flask db upgrade" --name api_db_migration
 
 .PHONY: cf-check-api-db-migration-task
 cf-check-api-db-migration-task: ## Get the status for the last notifications-api task
-	@cf curl /v3/apps/`cf app --guid notifications-api`/tasks?order_by=-created_at | jq -r ".resources[0].state"
+	@cf curl /v3/apps/`cf app --guid notify-api-alt`/tasks?order_by=-created_at | jq -r ".resources[0].state"
 
 .PHONY: cf-rollback
 cf-rollback: ## Rollbacks the app to the previous release

README.md

@@ -18,13 +18,14 @@ Install VS Code
 Open VS Code and install the Remote-Containers plug-in from Microsoft.
 
 Make sure your docker daemon is running (on OS X, this is typically accomplished by opening the Docker Desktop app)
+Also make sure there is NOT a Postgres daemon running on port 5432.
 
 Create the external docker network:
 
 `docker network create notify-network`
 
 Using the command palette (shift+cmd+p), search and select “Remote Containers: Open Folder in Container...”
-When prompted, choose **devcontainer-api** folder (note: this is a *subfolder* of notification-api). This will startup the container in a new window (replacing the current one). 
+When prompted, choose **devcontainer-api** folder (note: this is a *subfolder* of notification-api). This will startup the container in a new window (replacing the current one).
 
 After this page loads, hit "show logs” in bottom-right. The first time this runs it will need to build the Docker image, which will likely take several minutes.
 
@@ -34,12 +35,14 @@ Select View->Open View..., then search/select “ports”. Await a green dot on
 Open another terminal and run the background tasks:
 `make run-celery`
 
+Confirm that everything is working by hitting localhost:6011 and it responds with a 200 OK.
+
 ---
 ## Setting Up
 
 ### `.env` file
 
-Create and edit a .env file, based on sample.env. 
+Create and edit a .env file, based on sample.env.
 
 NOTE: when you change .env in the future, you'll need to rebuild the devcontainer for the change to take effect. Vscode _should_ detect the change and prompt you with a toast notification during a cached build. If not, you can find a manual rebuild in command pallette or just `docker rm` the notifications-api container.
 
@@ -109,10 +112,19 @@ make bootstrap
 make test
 ```
 
+## To run a local OWASP scan
+
+1. Run `make run-flask` from within the dev container.
+2. On your host machine run:
+
+```
+docker run -v $(pwd):/zap/wrk/:rw --network="notify-network" -t owasp/zap2docker-weekly zap-api-scan.py -t http://dev:6011/_status -f openapi -c zap.conf
+```
+
 ## To run scheduled tasks
 
 ```
-# After scheduling some tasks, open a third terminal in your running devcontainer and run celery beat 
+# After scheduling some tasks, open a third terminal in your running devcontainer and run celery beat
 make run-celery-beat
 ```
 

app/__init__.py

@@ -1,5 +1,5 @@
 import os
-import random
+import secrets
 import string
 import time
 import uuid
@@ -41,12 +41,13 @@ class SQLAlchemy(_SQLAlchemy):
     """We need to subclass SQLAlchemy in order to override create_engine options"""
 
     def apply_driver_hacks(self, app, info, options):
-        super().apply_driver_hacks(app, info, options)
+        sa_url, options = super().apply_driver_hacks(app, info, options)
         if 'connect_args' not in options:
             options['connect_args'] = {}
         options['connect_args']["options"] = "-c statement_timeout={}".format(
             int(app.config['SQLALCHEMY_STATEMENT_TIMEOUT']) * 1000
         )
+        return (sa_url, options)
 
 
 db = SQLAlchemy()
@@ -138,6 +139,7 @@ def register_blueprint(application):
     )
     from app.billing.rest import billing_blueprint
     from app.broadcast_message.rest import broadcast_message_blueprint
+    from app.celery.process_ses_receipts_tasks import ses_callback_blueprint
     from app.complaint.complaint_rest import complaint_blueprint
     from app.email_branding.rest import email_branding_blueprint
     from app.events.rest import events as events_blueprint
@@ -195,6 +197,10 @@ def register_blueprint(application):
 
     status_blueprint.before_request(requires_no_auth)
     application.register_blueprint(status_blueprint)
+    
+    # delivery receipts
+    ses_callback_blueprint.before_request(requires_no_auth)
+    application.register_blueprint(ses_callback_blueprint)
 
     # delivery receipts
     # TODO: make sure research mode can still trigger sms callbacks, then re-enable this
@@ -352,7 +358,7 @@ def create_uuid():
 
 
 def create_random_identifier():
-    return ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(16))
+    return ''.join(secrets.choice(string.ascii_uppercase + string.digits) for _ in range(16))
 
 
 def setup_sqlalchemy_events(app):

app/authentication/auth.py

@@ -18,7 +18,7 @@ from sqlalchemy.orm.exc import NoResultFound
 
 from app.serialised_models import SerialisedService
 
-GENERAL_TOKEN_ERROR_MESSAGE = 'Invalid token: make sure your API token matches the example at https://docs.notifications.service.gov.uk/rest-api.html#authorisation-header'  # noqa
+GENERAL_TOKEN_ERROR_MESSAGE = 'Invalid token: make sure your API token matches the example at https://docs.notifications.service.gov.uk/rest-api.html#authorisation-header'  # nosec B105
 
 AUTH_DB_CONNECTION_DURATION_SECONDS = Histogram(
     'auth_db_connection_duration_seconds',

app/celery/process_ses_receipts_tasks.py

@@ -1,80 +1,166 @@
+import enum
 from datetime import datetime, timedelta
+from json import decoder
 
 import iso8601
+import requests
+import traceback
 from celery.exceptions import Retry
-from flask import current_app, json
+from flask import Blueprint, current_app, json, jsonify, request
 from sqlalchemy.orm.exc import NoResultFound
 
-from app import notify_celery, statsd_client
+from app import notify_celery, statsd_client, redis_store
-from app.clients.email.aws_ses import get_aws_responses
+from app.celery.validate_sns import validate_sns_message
 from app.config import QueueNames
 from app.dao import notifications_dao
+from app.errors import InvalidRequest, register_errors
 from app.models import NOTIFICATION_PENDING, NOTIFICATION_SENDING
 from app.notifications.notifications_ses_callback import (
     _check_and_queue_complaint_callback_task,
     check_and_queue_callback_task,
     determine_notification_bounce_type,
+    get_aws_responses,
     handle_complaint,
 )
 
+ses_callback_blueprint = Blueprint('notifications_ses_callback', __name__)
+DEFAULT_MAX_AGE = timedelta(days=10000)
+
+register_errors(ses_callback_blueprint)
+class SNSMessageType(enum.Enum):
+    SubscriptionConfirmation = 'SubscriptionConfirmation'
+    Notification = 'Notification'
+    UnsubscribeConfirmation = 'UnsubscribeConfirmation'
+
+
+class InvalidMessageTypeException(Exception):
+    pass
+
+
+def verify_message_type(message_type: str):
+    try:
+        SNSMessageType(message_type)
+    except ValueError:
+        raise InvalidMessageTypeException(f'{message_type} is not a valid message type.')
+
+
+# 400 counts as a permanent failure so SNS will not retry.
+# 500 counts as a failed delivery attempt so SNS will retry.
+# See https://docs.aws.amazon.com/sns/latest/dg/DeliveryPolicies.html#DeliveryPolicies
+# This should not be here, it used to be in notifications/notifications_ses_callback. It then
+# got refactored into a task, which is fine, but it created a circular dependency. Will need
+# to investigate why GDS extracted this into a lambda
+@ses_callback_blueprint.route('/notifications/email/ses', methods=['POST'])
+def sns_callback_handler():
+    message_type = request.headers.get('x-amz-sns-message-type')
+    try:
+        verify_message_type(message_type)
+    except InvalidMessageTypeException:
+        current_app.logger.exception(f"Response headers: {request.headers}\nResponse data: {request.data}")
+        raise InvalidRequest("SES-SNS callback failed: invalid message type", 400)
+
+    try:
+        message = json.loads(request.data.decode('utf-8'))
+    except decoder.JSONDecodeError:
+        current_app.logger.exception(f"Response headers: {request.headers}\nResponse data: {request.data}")
+        raise InvalidRequest("SES-SNS callback failed: invalid JSON given", 400)
+
+    try:
+        validate_sns_message(message)
+    except Exception as err:
+        current_app.logger.error(f"SES-SNS callback failed: validation failed! Response headers: {request.headers}\nResponse data: {request.data}\nError: Signature validation failed with error {err}")
+        raise InvalidRequest("SES-SNS callback failed: validation failed", 400)
+
+    if message.get('Type') == 'SubscriptionConfirmation':
+        url = message.get('SubscribeUrl') if 'SubscribeUrl' in message else message.get('SubscribeURL')
+        response = requests.get(url)
+        try:
+            response.raise_for_status()
+        except Exception as e:
+            current_app.logger.warning(f"Attempt to raise_for_status()SubscriptionConfirmation Type message files for response: {response.text} with error {e}")
+            raise e
+
+        return jsonify(
+            result="success", message="SES-SNS auto-confirm callback succeeded"
+        ), 200
+
+    # TODO remove after smoke testing on prod is implemented
+    current_app.logger.info(f"SNS message: {message} is a valid delivery status message. Attempting to process it now.")
+    
+    process_ses_results.apply_async([{"Message": message.get("Message")}], queue=QueueNames.NOTIFY)
+
+    return jsonify(
+        result="success", message="SES-SNS callback succeeded"
+    ), 200
+
 
 @notify_celery.task(bind=True, name="process-ses-result", max_retries=5, default_retry_delay=300)
 def process_ses_results(self, response):
     try:
-        ses_message = json.loads(response['Message'])
+        ses_message = json.loads(response["Message"])
-        notification_type = ses_message['notificationType']
+        notification_type = ses_message["notificationType"]
+        # TODO remove after smoke testing on prod is implemented
+        current_app.logger.info(f"Attempting to process SES delivery status message from SNS with type: {notification_type} and body: {ses_message}")
         bounce_message = None
 
         if notification_type == 'Bounce':
-            notification_type, bounce_message = determine_notification_bounce_type(notification_type, ses_message)
+            bounce_message = determine_notification_bounce_type(ses_message)
         elif notification_type == 'Complaint':
             _check_and_queue_complaint_callback_task(*handle_complaint(ses_message))
             return True
 
-        aws_response_dict = get_aws_responses(notification_type)
+        aws_response_dict = get_aws_responses(ses_message)
 
-        notification_status = aws_response_dict['notification_status']
+        notification_status = aws_response_dict["notification_status"]
-        reference = ses_message['mail']['messageId']
+        reference = ses_message["mail"]["messageId"]
 
         try:
-            notification = notifications_dao.dao_get_notification_or_history_by_reference(reference=reference)
+            notification = notifications_dao.dao_get_notification_by_reference(reference)
         except NoResultFound:
-            message_time = iso8601.parse_date(ses_message['mail']['timestamp']).replace(tzinfo=None)
+            message_time = iso8601.parse_date(ses_message["mail"]["timestamp"]).replace(tzinfo=None)
             if datetime.utcnow() - message_time < timedelta(minutes=5):
                 current_app.logger.info(
-                    f"notification not found for reference: {reference} (update to {notification_status}). "
+                    f"notification not found for reference: {reference} (while attempting update to {notification_status}). "
                     f"Callback may have arrived before notification was persisted to the DB. Adding task to retry queue"
                 )
                 self.retry(queue=QueueNames.RETRY)
             else:
                 current_app.logger.warning(
-                    f"notification not found for reference: {reference} (update to {notification_status})"
+                    "notification not found for reference: {} (while attempting update to {})".format(reference, notification_status)
                 )
             return
 
         if bounce_message:
             current_app.logger.info(f"SES bounce for notification ID {notification.id}: {bounce_message}")
 
-        if notification.status not in [NOTIFICATION_SENDING, NOTIFICATION_PENDING]:
+        if notification.status not in {NOTIFICATION_SENDING, NOTIFICATION_PENDING}:
             notifications_dao._duplicate_update_warning(
-                notification=notification,
+                notification, 
-                status=notification_status
+                notification_status
             )
             return
+
+        notifications_dao._update_notification_status(
+            notification=notification,
+            status=notification_status,
+            provider_response=aws_response_dict["provider_response"],
+        )
+
+        if not aws_response_dict["success"]:
+            current_app.logger.info(
+                "SES delivery failed: notification id {} and reference {} has error found. Status {}".format(
+                    notification.id, reference, aws_response_dict["message"]
+                )
+            )
         else:
-            notifications_dao.dao_update_notifications_by_reference(
+            current_app.logger.info(
-                references=[reference],
+                "SES callback return status of {} for notification: {}".format(notification_status, notification.id)
-                update_dict={'status': notification_status}
             )
 
-        statsd_client.incr('callback.ses.{}'.format(notification_status))
+        statsd_client.incr("callback.ses.{}".format(notification_status))
 
         if notification.sent_at:
-            statsd_client.timing_with_dates(
+            statsd_client.timing_with_dates("callback.ses.elapsed-time", datetime.utcnow(), notification.sent_at)
-                f'callback.ses.{notification_status}.elapsed-time',
-                datetime.utcnow(),
-                notification.sent_at
-            )
 
         check_and_queue_callback_task(notification)
 
@@ -84,5 +170,6 @@ def process_ses_results(self, response):
         raise
 
     except Exception as e:
-        current_app.logger.exception('Error processing SES results: {}'.format(type(e)))
+        current_app.logger.exception("Error processing SES results: {}".format(type(e)))
         self.retry(queue=QueueNames.RETRY)
+

app/celery/process_sms_client_response_tasks.py

@@ -14,13 +14,13 @@ from app.notifications.notifications_ses_callback import (
     check_and_queue_callback_task,
 )
 
-# sms_response_mapper = {
+sms_response_mapper = {
-#     'MMG': get_mmg_responses,
+    # 'MMG': get_mmg_responses,
-#     'Firetext': get_firetext_responses,
+    # 'Firetext': get_firetext_responses,
-# }
+}
 
 
-gUpdate with new providers")
+# gUpdate with new providers")
 @notify_celery.task(bind=True, name="process-sms-client-response", max_retries=5, default_retry_delay=300)
 def process_sms_client_response(self, status, provider_reference, client_name, detailed_status_code=None):
     # validate reference

app/celery/research_mode_tasks.py

@@ -124,7 +124,7 @@ def create_fake_letter_response_file(self, reference):
     dvla_response_data = '{}|Sent|0|Sorted'.format(reference)
 
     # try and find a filename that hasn't been taken yet - from a random time within the last 30 seconds
-    for i in sorted(range(30), key=lambda _: random.random()):
+    for i in sorted(range(30), key=lambda _: random.random()): # nosec B311 - not security related
         upload_file_name = 'NOTIFY-{}-RSP.TXT'.format((now - timedelta(seconds=i)).strftime('%Y%m%d%H%M%S'))
         if not file_exists(current_app.config['DVLA_RESPONSE_BUCKET_NAME'], upload_file_name):
             break

app/celery/tasks.py

@@ -21,7 +21,7 @@ from app.dao.inbound_sms_dao import dao_get_inbound_sms_by_id
 from app.dao.jobs_dao import dao_get_job_by_id, dao_update_job
 from app.dao.notifications_dao import (
     dao_get_last_notification_added_for_job_id,
-    dao_get_notification_or_history_by_reference,
+    dao_get_notification_history_by_reference,
     dao_update_notifications_by_reference,
     get_notification_by_id,
     update_notification_status_by_reference,
@@ -547,7 +547,7 @@ def update_letter_notification(filename, temporary_failures, update):
 
 
 def check_billable_units(notification_update):
-    notification = dao_get_notification_or_history_by_reference(notification_update.reference)
+    notification = dao_get_notification_history_by_reference(notification_update.reference)
 
     if int(notification_update.page_count) != notification.billable_units:
         msg = 'Notification with id {} has {} billable_units but DVLA says page count is {}'.format(

app/celery/validate_sns.py

@@ -0,0 +1,113 @@
+import base64
+import re
+from urllib.parse import urlparse
+
+import requests
+import oscrypto.asymmetric
+import oscrypto.errors
+
+from app import redis_store
+from app.config import Config
+
+import six
+
+USE_CACHE = True
+VALIDATE_ARN = True
+VALID_SNS_TOPICS = Config.VALID_SNS_TOPICS
+
+
+_signing_cert_cache = {}
+_cert_url_re = re.compile(
+    r'sns\.([a-z]{1,3}-[a-z]+-[0-9]{1,2})\.amazonaws\.com',
+)
+
+class ValidationError(Exception):
+    """
+    ValidationError. Raised when a message fails integrity checks.
+    """
+
+
+def get_certificate(url):
+    if USE_CACHE:
+        res = redis_store.get(url)
+        if res is not None:
+            return res
+        res = requests.get(url).text
+        redis_store.set(url, res, ex=60 * 60)  # 60 minutes
+        return res
+    else:
+        return requests.get(url).text
+
+
+def validate_arn(sns_payload):
+    if VALIDATE_ARN:
+        arn = sns_payload.get('TopicArn')
+        topic_name = arn.split(':')[5]
+        if topic_name not in VALID_SNS_TOPICS:
+            raise ValidationError("Invalid Topic Name")
+
+
+def get_string_to_sign(sns_payload):
+    payload_type = sns_payload.get('Type')
+    if payload_type in ['SubscriptionConfirmation', 'UnsubscribeConfirmation']:
+        fields = ['Message', 'MessageId', 'SubscribeURL', 'Timestamp', 'Token', 'TopicArn', 'Type']
+    elif payload_type == 'Notification':
+        fields = ['Message', 'MessageId', 'Subject', 'Timestamp', 'TopicArn', 'Type']
+    else:
+        raise ValidationError("Unexpected Message Type")
+
+    string_to_sign = ''
+    for field in fields:
+        field_value = sns_payload.get(field)
+        if not isinstance(field_value, str):
+            if field == 'Subject' and field_value == None:
+                continue
+            raise ValidationError(f"In {field}, found non-string value: {field_value}")
+        string_to_sign += field + '\n' + field_value + '\n'
+    if isinstance(string_to_sign, six.text_type):
+        string_to_sign = string_to_sign.encode()
+    return string_to_sign
+
+
+def validate_sns_message(sns_payload):
+    """
+    Adapted from the solution posted at
+    https://github.com/boto/boto3/issues/2508#issuecomment-992931814
+    """
+    if not isinstance(sns_payload, dict):
+        raise ValidationError("Unexpected message type {!r}".format(type(sns_payload).__name__))
+
+    # Amazon SNS currently supports signature version 1.
+    if sns_payload.get('SignatureVersion') != '1':
+        raise ValidationError("Wrong Signature Version (expected 1)")
+    
+    validate_arn(sns_payload)
+    
+    string_to_sign = get_string_to_sign(sns_payload)
+
+    # Key signing cert url via Lambda and via webhook are slightly different
+    signing_cert_url = sns_payload.get('SigningCertUrl') if 'SigningCertUrl' in sns_payload else sns_payload.get('SigningCertURL')
+    if not isinstance(signing_cert_url, str):
+        raise ValidationError("Signing cert url must be a string")
+    cert_scheme, cert_netloc, *_ = urlparse(signing_cert_url)
+    if cert_scheme != 'https' or not re.match(_cert_url_re, cert_netloc):
+        raise ValidationError("Cert does not appear to be from AWS")
+    
+    certificate = _signing_cert_cache.get(signing_cert_url)
+    if certificate is None:
+        certificate = get_certificate(signing_cert_url)
+    if isinstance(certificate, six.text_type):
+        certificate = certificate.encode()
+    
+    signature = base64.b64decode(sns_payload["Signature"])
+
+    try:
+        oscrypto.asymmetric.rsa_pkcs1v15_verify(
+            oscrypto.asymmetric.load_certificate(certificate),
+            signature,
+            string_to_sign,
+            "sha1"
+        )
+        return True
+    except oscrypto.errors.SignatureError:
+        raise ValidationError("Invalid signature")

app/clients/__init__.py

@@ -38,8 +38,8 @@ class NotificationProviderClients(object):
         return self.email_clients.get(name)
 
     def get_client_by_name_and_type(self, name, notification_type):
-        assert notification_type in ['email', 'sms']
+        assert notification_type in ['email', 'sms'] # nosec B101
-        
+
         if notification_type == 'email':
             return self.get_email_client(name)
 

app/cloudfoundry_config.py

@@ -6,7 +6,6 @@ def extract_cloudfoundry_config():
     vcap_services = json.loads(os.environ['VCAP_SERVICES'])
 
     # Postgres config
-    os.environ['SQLALCHEMY_DATABASE_URI'] = vcap_services['aws-rds'][0]['credentials']['uri'].replace('postgres',
+    os.environ['SQLALCHEMY_DATABASE_URI'] = vcap_services['aws-rds'][0]['credentials']['uri'].replace('postgres','postgresql')
-                                                                                                      'postgresql')
     # Redis config
-    os.environ['REDIS_URL'] = vcap_services['aws-elasticache-redis'][0]['credentials']['uri']
+    os.environ['REDIS_URL'] = vcap_services['aws-elasticache-redis'][0]['credentials']['uri'].replace('redis://','rediss://')

app/commands.py

@@ -151,8 +151,8 @@ def backfill_notification_statuses():
     `Notification._status_enum`
     """
     LIMIT = 250000
-    subq = "SELECT id FROM notification_history WHERE notification_status is NULL LIMIT {}".format(LIMIT)
+    subq = "SELECT id FROM notification_history WHERE notification_status is NULL LIMIT {}".format(LIMIT) # nosec B608 no user-controlled input
-    update = "UPDATE notification_history SET notification_status = status WHERE id in ({})".format(subq)
+    update = "UPDATE notification_history SET notification_status = status WHERE id in ({})".format(subq) # nosec B608 no user-controlled input
     result = db.session.execute(subq).fetchall()
 
     while len(result) > 0:
@@ -169,7 +169,7 @@ def update_notification_international_flag():
     """
     # 250,000 rows takes 30 seconds to update.
     subq = "select id from notifications where international is null limit 250000"
-    update = "update notifications set international = False where id in ({})".format(subq)
+    update = "update notifications set international = False where id in ({})".format(subq) # nosec B608 no user-controlled input
     result = db.session.execute(subq).fetchall()
 
     while len(result) > 0:
@@ -180,7 +180,7 @@ def update_notification_international_flag():
 
     # Now update notification_history
     subq_history = "select id from notification_history where international is null limit 250000"
-    update_history = "update notification_history set international = False where id in ({})".format(subq_history)
+    update_history = "update notification_history set international = False where id in ({})".format(subq_history) # nosec B608 no user-controlled input
     result_history = db.session.execute(subq_history).fetchall()
     while len(result_history) > 0:
         db.session.execute(update_history)
@@ -201,8 +201,8 @@ def fix_notification_statuses_not_in_sync():
     """
     MAX = 10000
 
-    subq = "SELECT id FROM notifications WHERE cast (status as text) != notification_status LIMIT {}".format(MAX)
+    subq = "SELECT id FROM notifications WHERE cast (status as text) != notification_status LIMIT {}".format(MAX) # nosec B608 no user-controlled input
-    update = "UPDATE notifications SET notification_status = status WHERE id in ({})".format(subq)
+    update = "UPDATE notifications SET notification_status = status WHERE id in ({})".format(subq) # nosec B608 no user-controlled input
     result = db.session.execute(subq).fetchall()
 
     while len(result) > 0:
@@ -211,9 +211,8 @@ def fix_notification_statuses_not_in_sync():
         db.session.commit()
         result = db.session.execute(subq).fetchall()
 
-    subq_hist = "SELECT id FROM notification_history WHERE cast (status as text) != notification_status LIMIT {}" \
+    subq_hist = "SELECT id FROM notification_history WHERE cast (status as text) != notification_status LIMIT {}".format(MAX)  # nosec B608
-        .format(MAX)
+    update = "UPDATE notification_history SET notification_status = status WHERE id in ({})".format(subq_hist) # nosec B608 no user-controlled input
-    update = "UPDATE notification_history SET notification_status = status WHERE id in ({})".format(subq_hist)
     result = db.session.execute(subq_hist).fetchall()
 
     while len(result) > 0:
@@ -545,7 +544,8 @@ def populate_organisation_agreement_details_from_file(file_name):
 
             current_app.logger.info(f"Updating {org.name}")
 
-            assert org.agreement_signed
+            if not org.agreement_signed:
+                raise RuntimeError('Agreement was not signed')
 
             org.agreement_signed_version = float(row[1])
             org.agreement_signed_on_behalf_of_name = row[2].strip()

app/config.py

@@ -102,7 +102,7 @@ class Config(object):
 
     # DB conection string
     SQLALCHEMY_DATABASE_URI = os.environ.get('SQLALCHEMY_DATABASE_URI')
-    
+
     # AWS SMS
     AWS_PINPOINT_REGION = os.environ.get("AWS_PINPOINT_REGION")
     AWS_US_TOLL_FREE_NUMBER = os.environ.get("AWS_US_TOLL_FREE_NUMBER")
@@ -116,6 +116,12 @@ class Config(object):
 
     # Prefix to identify queues in SQS
     NOTIFICATION_QUEUE_PREFIX = os.environ.get('NOTIFICATION_QUEUE_PREFIX')
+    
+    # Use notify.sandbox.10x sending domain unless overwritten by environment
+    NOTIFY_EMAIL_DOMAIN = 'notify.sandbox.10x.gsa.gov'
+    
+    # AWS SNS topics for delivery receipts
+    VALID_SNS_TOPICS = ['notify_test_bounce', 'notify_test_success', 'notify_test_complaint']
 
     # URL of redis instance
     REDIS_URL = os.environ.get('REDIS_URL')
@@ -157,7 +163,7 @@ class Config(object):
     ONE_OFF_MESSAGE_FILENAME = 'Report'
     MAX_VERIFY_CODE_COUNT = 5
     MAX_FAILED_LOGIN_COUNT = 10
-    
+
     SES_STUB_URL = None # TODO: set to a URL in env and remove this to use a stubbed SES service
 
     # be careful increasing this size without being sure that we won't see slowness in pysftp
@@ -179,7 +185,7 @@ class Config(object):
     SMS_CODE_TEMPLATE_ID = '36fb0730-6259-4da1-8a80-c8de22ad4246'
     EMAIL_2FA_TEMPLATE_ID = '299726d2-dba6-42b8-8209-30e1d66ea164'
     NEW_USER_EMAIL_VERIFICATION_TEMPLATE_ID = 'ece42649-22a8-4d06-b87f-d52d5d3f0a27'
-    PASSWORD_RESET_TEMPLATE_ID = '474e9242-823b-4f99-813d-ed392e7f1201'
+    PASSWORD_RESET_TEMPLATE_ID = '474e9242-823b-4f99-813d-ed392e7f1201' # nosec B105 - this is not a password
     ALREADY_REGISTERED_EMAIL_TEMPLATE_ID = '0880fbb1-a0c6-46f0-9a8e-36c986381ceb'
     CHANGE_EMAIL_CONFIRMATION_TEMPLATE_ID = 'eb4d9930-87ab-4aef-9bce-786762687884'
     SERVICE_NOW_LIVE_TEMPLATE_ID = '618185c6-3636-49cd-b7d2-6f6f5eb3bdde'
@@ -426,7 +432,7 @@ class Development(Config):
     #     Config.GOVUK_ALERTS_CLIENT_ID: ['govuk-alerts-secret-key']
     # }
 
-    SECRET_KEY = 'dev-notify-secret-key'
+    SECRET_KEY = 'dev-notify-secret-key' # nosec B105 - this is only used in development
     DANGEROUS_SALT = 'dev-notify-salt'
 
     MMG_INBOUND_SMS_AUTH = ['testkey']
@@ -434,7 +440,8 @@ class Development(Config):
 
     NOTIFY_ENVIRONMENT = 'development'
     NOTIFY_LOG_PATH = 'application.log'
-    NOTIFY_EMAIL_DOMAIN = "dispostable.com"
+    
+    NOTIFY_EMAIL_DOMAIN = os.getenv('NOTIFY_EMAIL_DOMAIN', 'notify.sandbox.10x.gsa.gov')
 
     SQLALCHEMY_DATABASE_URI = os.environ.get('SQLALCHEMY_DATABASE_URI', 'postgresql://postgres:chummy@db:5432/notification_api')
     REDIS_URL = os.environ.get('REDIS_URL')
@@ -533,7 +540,6 @@ class Staging(Config):
 
 
 class Live(Config):
-    NOTIFY_EMAIL_DOMAIN = os.environ.get('NOTIFY_EMAIL_DOMAIN')
     NOTIFY_ENVIRONMENT = 'live'
     # buckets
     CSV_UPLOAD_BUCKET_NAME = 'notifications-prototype-csv-upload' # created in gsa sandbox
@@ -546,13 +552,13 @@ class Live(Config):
     INVALID_PDF_BUCKET_NAME = 'production-letters-invalid-pdf' # not created in gsa sandbox
     TRANSIENT_UPLOADED_LETTERS = 'production-transient-uploaded-letters' # not created in gsa sandbox
     LETTER_SANITISE_BUCKET_NAME = 'production-letters-sanitise' # not created in gsa sandbox
-    
+
     FROM_NUMBER = 'US Notify'
     API_RATE_LIMIT_ENABLED = True
     CHECK_PROXY_HEADER = True
     SES_STUB_URL = None
     CRONITOR_ENABLED = True
-    
+
     # DEBUG = True
     REDIS_ENABLED = os.environ.get('REDIS_ENABLED')
 

app/dao/notifications_dao.py

@@ -87,16 +87,21 @@ def country_records_delivery(phone_prefix):
     dlr = INTERNATIONAL_BILLING_RATES[phone_prefix]['attributes']['dlr']
     return dlr and dlr.lower() == 'yes'
 
+def _decide_permanent_temporary_failure(current_status, status):
+    # If we go from pending to delivered we need to set failure type as temporary-failure
+    if current_status == NOTIFICATION_PENDING and status == NOTIFICATION_PERMANENT_FAILURE:
+        status = NOTIFICATION_TEMPORARY_FAILURE
+    return status
 
-def _update_notification_status(notification, status, detailed_status_code=None):
+
-    # status = _decide_permanent_temporary_failure(
+def _update_notification_status(notification, status, provider_response=None):
-    #     status=status, notification=notification, detailed_status_code=detailed_status_code
+    status = _decide_permanent_temporary_failure(current_status=notification.status, status=status)
-    # )
+    notification.status = status
-    # notification.status = status
+    if provider_response:
-    # dao_update_notification(notification)
+        notification.provider_response = provider_response
+    dao_update_notification(notification)
     return notification
 
-
 @autocommit
 def update_notification_status_by_id(notification_id, status, sent_by=None, detailed_status_code=None):
     notification = Notification.query.with_for_update().filter(Notification.id == notification_id).first()
@@ -587,17 +592,13 @@ def dao_get_notification_by_reference(reference):
     ).one()
 
 
-def dao_get_notification_or_history_by_reference(reference):
+def dao_get_notification_history_by_reference(reference):
     try:
         # This try except is necessary because in test keys and research mode does not create notification history.
         # Otherwise we could just search for the NotificationHistory object
-        return Notification.query.filter(
+        return Notification.query.filter(Notification.reference == reference).one()
-            Notification.reference == reference
-        ).one()
     except NoResultFound:
-        return NotificationHistory.query.filter(
+        return NotificationHistory.query.filter(NotificationHistory.reference == reference).one()
-            NotificationHistory.reference == reference
-        ).one()
 
 
 def dao_get_notifications_processing_time_stats(start_date, end_date):

app/delivery/send_to_providers.py

@@ -163,7 +163,10 @@ def update_notification_to_sending(notification, provider):
     notification.sent_at = datetime.utcnow()
     notification.sent_by = provider.name
     if notification.status not in NOTIFICATION_STATUS_TYPES_COMPLETED:
-        notification.status = NOTIFICATION_SENT if notification.international else NOTIFICATION_SENDING
+        # We currently have no callback method for SMS deliveries
+        # TODO create celery task to request SMS delivery receipts from cloudwatch api
+        notification.status = NOTIFICATION_SENT if notification.notification_type == "sms" else NOTIFICATION_SENDING
+    
     dao_update_notification(notification)
 
 
@@ -188,7 +191,7 @@ def provider_to_use(notification_type, international=True):
         chosen_provider = active_providers[0]
     else:
         weights = [p.priority for p in active_providers]
-        chosen_provider = random.choices(active_providers, weights=weights)[0]
+        chosen_provider = random.choices(active_providers, weights=weights)[0] # nosec B311 - this is not security/cryptography related
 
     return notification_provider_clients.get_client_by_name_and_type(chosen_provider.identifier, notification_type)
 

app/models.py

@@ -1505,6 +1505,8 @@ class Notification(db.Model):
     document_download_count = db.Column(db.Integer, nullable=True)
 
     postage = db.Column(db.String, nullable=True)
+    provider_response = db.Column(db.Text, nullable=True)
+    # queue_name = db.Column(db.Text, nullable=True)
 
     __table_args__ = (
         db.ForeignKeyConstraint(
@@ -1647,7 +1649,7 @@ class Notification(db.Model):
         """
         # this should only ever be called for letter notifications - it makes no sense otherwise and I'd rather not
         # get the two code flows mixed up at all
-        assert self.notification_type == LETTER_TYPE
+        assert self.notification_type == LETTER_TYPE # nosec B101 - current calling code already validates the correct type
 
         if self.status in [NOTIFICATION_CREATED, NOTIFICATION_SENDING]:
             return NOTIFICATION_STATUS_LETTER_ACCEPTED
@@ -1707,6 +1709,7 @@ class Notification(db.Model):
             "postcode": None,
             "type": self.notification_type,
             "status": self.get_letter_status() if self.notification_type == LETTER_TYPE else self.status,
+            "provider_response": self.provider_response,
             "template": template_dict,
             "body": self.content,
             "subject": self.subject,

app/notifications/callbacks.py

@@ -0,0 +1,52 @@
+from app.celery.service_callback_tasks import send_delivery_status_to_service
+from app.config import QueueNames
+from app.dao.service_callback_api_dao import (
+    get_service_delivery_status_callback_api_for_service,
+)
+
+
+def check_and_queue_callback_task(notification):
+    # queue callback task only if the service_callback_api exists
+    service_callback_api = get_service_delivery_status_callback_api_for_service(service_id=notification.service_id)
+    if service_callback_api:
+        notification_data = create_delivery_status_callback_data(notification, service_callback_api)
+
+        send_delivery_status_to_service.apply_async([str(notification.id), notification_data], queue=QueueNames.CALLBACKS)
+
+
+def create_delivery_status_callback_data(notification, service_callback_api):
+    from app import encryption
+    from app.utils import DATETIME_FORMAT
+
+    data = {
+        "notification_id": str(notification.id),
+        "notification_client_reference": notification.client_reference,
+        "notification_to": notification.to,
+        "notification_status": notification.status,
+        "notification_provider_response": notification.provider_response,
+        "notification_created_at": notification.created_at.strftime(DATETIME_FORMAT),
+        "notification_updated_at": notification.updated_at.strftime(DATETIME_FORMAT) if notification.updated_at else None,
+        "notification_sent_at": notification.sent_at.strftime(DATETIME_FORMAT) if notification.sent_at else None,
+        "notification_type": notification.notification_type,
+        "service_callback_api_url": service_callback_api.url,
+        "service_callback_api_bearer_token": service_callback_api.bearer_token,
+    }
+
+    return encryption.encrypt(data)
+
+
+def create_complaint_callback_data(complaint, notification, service_callback_api, recipient):
+    from app import encryption
+    from app.utils import DATETIME_FORMAT
+
+    data = {
+        "complaint_id": str(complaint.id),
+        "notification_id": str(notification.id),
+        "reference": notification.client_reference,
+        "to": recipient,
+        "complaint_date": complaint.complaint_date.strftime(DATETIME_FORMAT),
+        "service_callback_api_url": service_callback_api.url,
+        "service_callback_api_bearer_token": service_callback_api.bearer_token,
+    }
+
+    return encryption.encrypt(data)

app/notifications/notifications_ses_callback.py

@@ -1,4 +1,4 @@
-from flask import current_app
+from flask import current_app, json
 
 from app.celery.service_callback_tasks import (
     create_complaint_callback_data,
@@ -8,64 +8,118 @@ from app.celery.service_callback_tasks import (
 )
 from app.config import QueueNames
 from app.dao.complaint_dao import save_complaint
-from app.dao.notifications_dao import (
+from app.dao.notifications_dao import dao_get_notification_history_by_reference
-    dao_get_notification_or_history_by_reference,
-)
 from app.dao.service_callback_api_dao import (
     get_service_complaint_callback_api_for_service,
     get_service_delivery_status_callback_api_for_service,
 )
 from app.models import Complaint
+from app.notifications.callbacks import create_complaint_callback_data
 
 
-def determine_notification_bounce_type(notification_type, ses_message):
+def determine_notification_bounce_type(ses_message):
+    notification_type = ses_message["notificationType"]
+    if notification_type in ["Delivery", "Complaint"]:
+        return notification_type
+
+    if notification_type != "Bounce":
+        raise KeyError(f"Unhandled notification type {notification_type}")
+
     remove_emails_from_bounce(ses_message)
-    if ses_message['bounce']['bounceType'] == 'Permanent':
+    current_app.logger.info("SES bounce dict: {}".format(json.dumps(ses_message).replace("{", "(").replace("}", ")")))
-        notification_type = ses_message['bounce']['bounceType']  # permanent or not
+    if ses_message["bounce"]["bounceType"] == "Permanent":
-    else:
+        return "Permanent"
-        notification_type = 'Temporary'
+    return "Temporary"
-    return notification_type, ses_message
+
+
+def _determine_provider_response(ses_message):
+    if ses_message["notificationType"] != "Bounce":
+        return None
+
+    bounce_type = ses_message["bounce"]["bounceType"]
+    bounce_subtype = ses_message["bounce"]["bounceSubType"]
+
+    # See https://docs.aws.amazon.com/ses/latest/DeveloperGuide/event-publishing-retrieving-sns-contents.html
+    if bounce_type == "Permanent" and bounce_subtype == "Suppressed":
+        return "The email address is on our email provider suppression list"
+    elif bounce_type == "Permanent" and bounce_subtype == "OnAccountSuppressionList":
+        return "The email address is on the GC Notify suppression list"
+    elif bounce_type == "Transient" and bounce_subtype == "AttachmentRejected":
+        return "The email was rejected because of its attachments"
+
+    return None
+
+
+def get_aws_responses(ses_message):
+    status = determine_notification_bounce_type(ses_message)
+
+    base = {
+        "Permanent": {
+            "message": "Hard bounced",
+            "success": False,
+            "notification_status": "permanent-failure",
+        },
+        "Temporary": {
+            "message": "Soft bounced",
+            "success": False,
+            "notification_status": "temporary-failure",
+        },
+        "Delivery": {
+            "message": "Delivered",
+            "success": True,
+            "notification_status": "delivered",
+        },
+        "Complaint": {
+            "message": "Complaint",
+            "success": True,
+            "notification_status": "delivered",
+        },
+    }[status]
+
+    base["provider_response"] = _determine_provider_response(ses_message)
+
+    return base
 
 
 def handle_complaint(ses_message):
     recipient_email = remove_emails_from_complaint(ses_message)[0]
-    current_app.logger.info("Complaint from SES: \n{}".format(ses_message))
+    current_app.logger.info("Complaint from SES: \n{}".format(json.dumps(ses_message).replace("{", "(").replace("}", ")")))
     try:
-        reference = ses_message['mail']['messageId']
+        reference = ses_message["mail"]["messageId"]
     except KeyError as e:
         current_app.logger.exception("Complaint from SES failed to get reference from message", e)
         return
-    notification = dao_get_notification_or_history_by_reference(reference)
+    notification = dao_get_notification_history_by_reference(reference)
-    ses_complaint = ses_message.get('complaint', None)
+    ses_complaint = ses_message.get("complaint", None)
 
     complaint = Complaint(
         notification_id=notification.id,
         service_id=notification.service_id,
-        ses_feedback_id=ses_complaint.get('feedbackId', None) if ses_complaint else None,
+        ses_feedback_id=ses_complaint.get("feedbackId", None) if ses_complaint else None,
-        complaint_type=ses_complaint.get('complaintFeedbackType', None) if ses_complaint else None,
+        complaint_type=ses_complaint.get("complaintFeedbackType", None) if ses_complaint else None,
-        complaint_date=ses_complaint.get('timestamp', None) if ses_complaint else None
+        complaint_date=ses_complaint.get("timestamp", None) if ses_complaint else None,
     )
     save_complaint(complaint)
     return complaint, notification, recipient_email
 
 
 def remove_mail_headers(dict_to_edit):
-    if dict_to_edit['mail'].get('headers'):
+    if dict_to_edit["mail"].get("headers"):
-        dict_to_edit['mail'].pop('headers')
+        dict_to_edit["mail"].pop("headers")
-    if dict_to_edit['mail'].get('commonHeaders'):
+    if dict_to_edit["mail"].get("commonHeaders"):
-        dict_to_edit['mail'].pop('commonHeaders')
+        dict_to_edit["mail"].pop("commonHeaders")
 
 
 def remove_emails_from_bounce(bounce_dict):
     remove_mail_headers(bounce_dict)
-    bounce_dict['mail'].pop('destination')
+    bounce_dict["mail"].pop("destination", None)
-    bounce_dict['bounce'].pop('bouncedRecipients')
+    bounce_dict["bounce"].pop("bouncedRecipients", None)
 
 
 def remove_emails_from_complaint(complaint_dict):
     remove_mail_headers(complaint_dict)
-    complaint_dict['complaint'].pop('complainedRecipients')
+    complaint_dict["complaint"].pop("complainedRecipients")
-    return complaint_dict['mail'].pop('destination')
+    return complaint_dict["mail"].pop("destination")
 
 
 def check_and_queue_callback_task(notification):
@@ -73,8 +127,7 @@ def check_and_queue_callback_task(notification):
     service_callback_api = get_service_delivery_status_callback_api_for_service(service_id=notification.service_id)
     if service_callback_api:
         notification_data = create_delivery_status_callback_data(notification, service_callback_api)
-        send_delivery_status_to_service.apply_async([str(notification.id), notification_data],
+        send_delivery_status_to_service.apply_async([str(notification.id), notification_data], queue=QueueNames.CALLBACKS)
-                                                    queue=QueueNames.CALLBACKS)
 
 
 def _check_and_queue_complaint_callback_task(complaint, notification, recipient):
@@ -83,3 +136,4 @@ def _check_and_queue_complaint_callback_task(complaint, notification, recipient)
     if service_callback_api:
         complaint_data = create_complaint_callback_data(complaint, notification, service_callback_api, recipient)
         send_complaint_to_service.apply_async([complaint_data], queue=QueueNames.CALLBACKS)
+        

app/notifications/receive_notifications.py

@@ -23,6 +23,62 @@ INBOUND_SMS_COUNTER = Counter(
     ['provider']
 )
 
+@receive_notifications_blueprint.route('/notifications/sms/receive/sns', methods=['POST'])
+def receive_sns_sms():
+    """
+    {
+        "originationNumber":"+14255550182",
+        "destinationNumber":"+12125550101",
+        "messageKeyword":"JOIN", # this is optional
+        "messageBody":"EXAMPLE",
+        "inboundMessageId":"cae173d2-66b9-564c-8309-21f858e9fb84",
+        "previousPublishedMessageId":"wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
+    }
+    """
+
+    post_data = request.get_json()
+
+    # validate sns from common module, WILL ALSO NEED TO AUTO-SUBSCRIBE... raise errors appropriately
+
+    # TODO modify this for AWS SNS
+    inbound_number = strip_leading_forty_four(post_data['Number'])
+
+    service = fetch_potential_service(inbound_number, 'sns')
+    if not service:
+        # since this is an issue with our service <-> number mapping, or no inbound_sms service permission
+        # we should still tell SNS that we received it successfully
+        current_app.logger.warning(f"Mapping between service id and inbound number is broken, or service does not have permission to receive inbound sms")
+        return jsonify({
+            "status": "ok"
+        }), 200 
+
+    INBOUND_SMS_COUNTER.labels("sns").inc()
+
+    content = format_mmg_message(post_data["Message"])
+    from_number = post_data['MSISDN']
+    provider_ref = post_data["ID"]
+    date_received = post_data.get('DateRecieved')
+    provider_name = "sns"
+
+    inbound_payload = {}
+
+    # TODO fill inbound_payload and spread like create_inbound_sms_object(service, **inbound_payload)
+    inbound = create_inbound_sms_object(service,
+                                        content=format_mmg_message(post_data["Message"]),
+                                        from_number=from_number,
+                                        provider_ref=provider_ref,
+                                        date_received=date_received,
+                                        provider_name=provider_name)
+
+    # TODO ensure inbound sms callback endpoints are accessible and functioning for notify api users
+    # tasks.send_inbound_sms_to_service.apply_async([str(inbound.id), str(service.id)], queue=QueueNames.NOTIFY)
+
+    current_app.logger.debug(
+        '{} received inbound SMS with reference {} from SNS'.format(service.id, inbound.provider_reference))
+
+    return jsonify({
+        "status": "ok"
+    }), 200 
 
 @receive_notifications_blueprint.route('/notifications/sms/receive/mmg', methods=['POST'])
 def receive_mmg_sms():

app/schema_validation/__init__.py

@@ -108,7 +108,7 @@ def __format_message(e):
             error_path = e.path.popleft()
             # no need to catch IndexError exception explicity as
             # error_path is None if e.path has no items
-        except Exception:
+        except IndexError:
             pass
         return error_path
 

app/xml_schemas/__init__.py

@@ -1,6 +1,8 @@
 from pathlib import Path
 
-from lxml import etree
+from defusedxml.lxml import fromstring
+# there is no equivalent in defusedxml to validate a schema
+from lxml.etree import XMLSchema  # nosec B410
 
 
 def validate_xml(document, schema_file_name):
@@ -8,13 +10,6 @@ def validate_xml(document, schema_file_name):
     path = Path(__file__).resolve().parent / schema_file_name
     contents = path.read_text()
 
-    schema_root = etree.XML(contents.encode('utf-8'))
+    schema_root = fromstring(contents.encode('utf-8'))
-    schema = etree.XMLSchema(schema_root)
+    schema = XMLSchema(schema_root)
-    parser = etree.XMLParser(schema=schema)
+    return schema.validate(fromstring(document))
-
-    try:
-        etree.fromstring(document, parser)
-    except etree.XMLSyntaxError:
-        return False
-
-    return True

devcontainer-api/.devcontainer.json

@@ -13,8 +13,11 @@
 		},
 		"python.linting.enabled": true,
 		"python.linting.pylintEnabled": true,
-		"python.pythonPath": "/usr/local/bin/python",
+		"python.defaultInterpreterPath": "/usr/bin/python3",
-		"python.linting.pylintPath": "/usr/local/share/pip-global/bin/pylint"
+		"python.linting.pylintPath": "/usr/local/share/pip-global/bin/pylint",
+		"python.analysis.extraPaths": [
+			"/home/vscode/.local/lib/python3.9/site-packages"
+		]
 	},
 	"features": {
 		"docker-from-docker": {
@@ -33,7 +36,6 @@
 		"visualstudioexptteam.vscodeintellicode",
 		"yzhang.markdown-all-in-one",
 		"ms-ossdata.vscode-postgresql",
-		"GitHub.copilot"
 		// "ms-vsliveshare.vsliveshare",
 		// "mtxr.sqltools",
 		// "mtxr.sqltools-driver-pg"

devcontainer-api/Dockerfile

@@ -21,6 +21,7 @@ RUN apt-get update \
     openssh-client \
     procps \
     sudo \
+    swig \
     tldr \
     unzip \
     vim \

manifest.yml

@@ -1,41 +1,44 @@
 ---
 
 applications:
-  - name: notifications-api
+  - name: notify-api-alt
-    buildpack: python_buildpack
+    buildpack: https://github.com/cloudfoundry/python-buildpack.git#v1.7.58
     instances: 1
     memory: 1G
     disk_quota: 1G
     health-check-type: process
     health-check-invocation-timeout: 1
     routes:
-    - route: notifications-api.app.cloud.gov
+    - route: notify-api-alt.app.cloud.gov
 
     services:
-      - api-psql
+      - api-alt-psql
-      - api-redis
+      - api-alt-redis
 
     env:
+      BP_PIP_VERSION: latest
       NOTIFY_APP_NAME: api
       NOTIFY_LOG_PATH: /home/vcap/logs/app.log
       FLASK_APP: application.py
       FLASK_ENV: production
 
       NOTIFY_ENVIRONMENT: live
-      API_HOST_NAME: https://notifications-api.app.cloud.gov
+      API_HOST_NAME: https://notify-api-alt.app.cloud.gov
-      ADMIN_BASE_URL: https://notifications-admin.app.cloud.gov
+      ADMIN_BASE_URL: https://notify-admin-alt.app.cloud.gov
-      NOTIFICATION_QUEUE_PREFIX: prototype_10x
+      NOTIFICATION_QUEUE_PREFIX: notify_alt_
+      REDIS_ENABLED: true
       STATSD_HOST: localhost
 
-      INTERNAL_CLIENT_API_KEYS: '{"notify-admin":["dev-notify-secret-key"]}'
+      INTERNAL_CLIENT_API_KEYS: '{"notify-admin":["((ADMIN_CLIENT_SECRET))"]}'
-      
+
       # Credentials variables
+      ADMIN_CLIENT_SECRET: ((ADMIN_CLIENT_SECRET))
       DANGEROUS_SALT: ((DANGEROUS_SALT))
       SECRET_KEY: ((SECRET_KEY))
+      AWS_ACCESS_KEY_ID: ((AWS_ACCESS_KEY_ID))
+      AWS_SECRET_ACCESS_KEY: ((AWS_SECRET_ACCESS_KEY))
       AWS_REGION: us-west-2
       AWS_PINPOINT_REGION: us-west-2
       AWS_US_TOLL_FREE_NUMBER: +18446120782
 
       DVLA_EMAIL_ADDRESSES: []
-
-      NOTIFY_EMAIL_DOMAIN: dispostable.com

manifest.yml.j2

@@ -1,155 +0,0 @@
-{%- set app_vars = {
-  'notify-api': {
-    'NOTIFY_APP_NAME': 'api',
-    'disk_quota': '2G',
-    'sqlalchemy_pool_size': 15,
-    'additional_env_vars': {
-      'STATSD_HOST': None
-    },
-    'routes': {
-      'preview': ['api.notify.works', 'notify-api-preview.apps.internal'],
-      'staging': ['api.staging-notify.works', 'notify-api-staging.apps.internal'],
-      'production': ['api.notifications.service.gov.uk', 'notify-api-production.apps.internal'],
-    },
-    'health-check-type': 'port',
-    'health-check-invocation-timeout': 3,
-    'instances': {
-      'preview': None,
-      'staging': None,
-      'production': 25
-    },
-  },
-
-  'notify-api-sms-receipts': {
-    'NOTIFY_APP_NAME': 'api',
-    'disk_quota': '2G',
-    'additional_env_vars': {
-      'STATSD_HOST': None
-    },
-    'routes': {
-      'preview': ['api.notify.works/notifications/sms/mmg', 'api.notify.works/notifications/sms/firetext', 'notify-api-sms-receipts-preview.apps.internal'],
-      'staging': ['api.staging-notify.works/notifications/sms/mmg', 'api.staging-notify.works/notifications/sms/firetext', 'notify-api-sms-receipts-staging.apps.internal'],
-      'production': ['api.notifications.service.gov.uk/notifications/sms/mmg', 'api.notifications.service.gov.uk/notifications/sms/firetext', 'notify-api-sms-receipts-production.apps.internal' ],
-    },
-    'health-check-type': 'port',
-    'health-check-invocation-timeout': 3,
-    'instances': {
-      'preview': 1,
-      'staging': 2,
-      'production': 10
-    },
-  },
-
-  'notify-api-db-migration': {
-    'NOTIFY_APP_NAME': 'api',
-    'instances': {
-      'preview': 0,
-      'staging': 0,
-      'production': 0
-    },
-  },
-
-  'notify-delivery-celery-beat': {'memory': '512M'},
-  'notify-delivery-worker-jobs': {'memory': '2G'},
-  'notify-delivery-worker-research': {},
-  'notify-delivery-worker-sender': {'disk_quota': '2G', 'memory': '4G'},
-  'notify-delivery-worker-periodic': {},
-  'notify-delivery-worker-reporting': {
-    'additional_env_vars': {
-      'CELERYD_PREFETCH_MULTIPLIER': 1,
-    }
-  },
-  'notify-delivery-worker-priority': {},
-  'notify-delivery-worker-letters': {'memory': '2G'},
-  'notify-delivery-worker-retry-tasks': {},
-  'notify-delivery-worker-internal': {},
-  'notify-delivery-worker-broadcasts': {
-    'additional_env_vars': {
-      'CELERYD_PREFETCH_MULTIPLIER': 1,
-      'CBC_PROXY_AWS_ACCESS_KEY_ID': CBC_PROXY_AWS_ACCESS_KEY_ID,
-      'CBC_PROXY_AWS_SECRET_ACCESS_KEY': CBC_PROXY_AWS_SECRET_ACCESS_KEY,
-    }
-  },
-  'notify-delivery-worker-receipts': {},
-  'notify-delivery-worker-service-callbacks': {'disk_quota': '2G'},
-  'notify-delivery-worker-save-api-notifications': {'disk_quota': '2G'},
-} -%}
-
-{%- set app = app_vars[CF_APP] -%}
-{%- set instance_count = app.get('instances', {}).get(environment) -%}
-
----
-
-applications:
-  - name: {{ CF_APP }}
-    buildpack: python_buildpack
-    {% if instance_count is not none %}
-    instances: {{ instance_count }}
-    {%- endif %}
-    memory: {{ app.get('memory', '1G') }}
-    disk_quota: {{ app.get('disk_quota', '1G')}}
-
-    routes:
-      {%- for route in app.get('routes', {}).get(environment, []) %}
-      - route: {{ route }}
-      {%- endfor%}
-      - route: {{ CF_APP }}-{{ environment }}.cloudapps.digital
-    health-check-type: {{ app.get('health-check-type', 'process') }}
-    health-check-invocation-timeout: {{ app.get('health-check-invocation-timeout', 1) }}
-
-    services:
-      - notify-db
-      - notify-redis
-      - logit-ssl-syslog-drain
-      {% if CF_APP == 'notify-api' %}
-      - notify-prometheus
-      - notify-splunk
-      {% endif %}
-
-    env:
-      NOTIFY_APP_NAME: {{ app.get('NOTIFY_APP_NAME', CF_APP.replace('notify-', '')) }}
-      NOTIFY_LOG_PATH: /home/vcap/logs/app.log
-      SQLALCHEMY_POOL_SIZE: {{ app.get('sqlalchemy_pool_size', 1) }}
-      FLASK_APP: application.py
-      NOTIFY_ENVIRONMENT: {{ environment }}
-
-      # Credentials variables
-      ADMIN_BASE_URL: '{{ ADMIN_BASE_URL }}'
-      INTERNAL_CLIENT_API_KEYS: '{{ INTERNAL_CLIENT_API_KEYS | tojson }}'
-      API_HOST_NAME: '{{ API_HOST_NAME }}'
-      DANGEROUS_SALT: '{{ DANGEROUS_SALT }}'
-      SECRET_KEY: '{{ SECRET_KEY }}'
-      ROUTE_SECRET_KEY_1: '{{ ROUTE_SECRET_KEY_1 }}'
-      ROUTE_SECRET_KEY_2: '{{ ROUTE_SECRET_KEY_2 }}'
-      CRONITOR_KEYS: '{{ CRONITOR_KEYS | tojson }}'
-      METRICS_BASIC_AUTH_TOKEN: {{ METRICS_BASIC_AUTH_TOKEN }}
-
-      HIGH_VOLUME_SERVICE: '{{ HIGH_VOLUME_SERVICE | tojson }}'
-
-      DOCUMENT_DOWNLOAD_API_HOST: '{{ DOCUMENT_DOWNLOAD_API_HOST }}'
-      DOCUMENT_DOWNLOAD_API_KEY: '{{ DOCUMENT_DOWNLOAD_API_KEY }}'
-
-      NOTIFICATION_QUEUE_PREFIX: '{{ NOTIFICATION_QUEUE_PREFIX }}'
-      AWS_ACCESS_KEY_ID: '{{ AWS_ACCESS_KEY_ID }}'
-      AWS_SECRET_ACCESS_KEY: '{{ AWS_SECRET_ACCESS_KEY }}'
-
-      STATSD_HOST: "notify-statsd-exporter-{{ environment }}.apps.internal"
-
-      ZENDESK_API_KEY: '{{ ZENDESK_API_KEY }}'
-
-      MMG_API_KEY: '{{ MMG_API_KEY }}'
-      MMG_INBOUND_SMS_AUTH: '{{ MMG_INBOUND_SMS_AUTH | tojson }}'
-      MMG_INBOUND_SMS_USERNAME: '{{ MMG_INBOUND_SMS_USERNAME | tojson }}'
-
-      FIRETEXT_API_KEY: '{{ FIRETEXT_API_KEY }}'
-      FIRETEXT_INTERNATIONAL_API_KEY: '{{ FIRETEXT_INTERNATIONAL_API_KEY }}'
-      FIRETEXT_INBOUND_SMS_AUTH: '{{ FIRETEXT_INBOUND_SMS_AUTH | tojson }}'
-
-      TEMPLATE_PREVIEW_API_HOST: '{{ TEMPLATE_PREVIEW_API_HOST }}'
-      TEMPLATE_PREVIEW_API_KEY: '{{ TEMPLATE_PREVIEW_API_KEY }}'
-
-      DVLA_EMAIL_ADDRESSES: '{{ DVLA_EMAIL_ADDRESSES | tojson  }}'
-
-      {% for key, value in app.get('additional_env_vars', {}).items() %}
-      {{key}}: '{{value}}'
-      {% endfor %}

migrations/versions/0374_fix_reg_template_history.py

@@ -0,0 +1,54 @@
+"""empty message
+
+Revision ID: 0374_fix_reg_template_history
+Revises: 0373_add_notifications_view
+Create Date: 2022-08-22 11:04:15.888017
+
+"""
+
+# revision identifiers, used by Alembic.
+from datetime import datetime
+
+revision = '0374_fix_reg_template_history'
+down_revision = '0373_add_notifications_view'
+
+from alembic import op
+import sqlalchemy as sa
+
+service_id = 'd6aa2c68-a2d9-4437-ab19-3ae8eb202553'
+user_id= '6af522d0-2915-4e52-83a3-3690455a5fe6'
+
+def upgrade():
+    op.get_bind()
+    
+    # modify subject of verification email in templates
+    table_name = 'templates'
+    col = 'subject'
+    val = 'Confirm US Notify registration'
+    select_by_col = 'name'
+    select_by_val = 'Notify email verification code'
+    op.execute(f"update {table_name} set {col}='{val}' where {select_by_col} = '{select_by_val}'")
+    
+    # modify subject of verification email in templates_history
+    table_name = 'templates_history'
+    op.execute(f"update {table_name} set {col}='{val}' where {select_by_col} = '{select_by_val}'")
+    
+    # modify content of verification email in templates
+    table_name = 'templates'
+    col = 'content'
+    val = """Hi ((name)),\n\nTo complete your registration for US Notify please click the link below\n\n((url))"""
+    select_by_col = 'name'
+    select_by_val = 'Notify email verification code'
+    op.execute(f"update {table_name} set {col}='{val}' where {select_by_col} = '{select_by_val}'")
+    
+    # modify content of verification email in templates_history
+    table_name = 'templates_history'
+    op.execute(f"update {table_name} set {col}='{val}' where {select_by_col} = '{select_by_val}'")
+    
+    # TODO: modify other templates as necessary and re-run this migration
+    
+
+def downgrade():
+    ### commands auto generated by Alembic - please adjust! ###
+    pass
+    ### end Alembic commands ###

migrations/versions/0375_fix_service_name.py

@@ -0,0 +1,39 @@
+"""empty message
+
+Revision ID: 0375_fix_service_name
+Revises: 0374_fix_reg_template_history
+Create Date: 2022-08-29 11:04:15.888017
+
+"""
+
+# revision identifiers, used by Alembic.
+from datetime import datetime
+
+revision = '0375_fix_service_name'
+down_revision = '0374_fix_reg_template_history'
+
+from alembic import op
+import sqlalchemy as sa
+
+service_id = 'd6aa2c68-a2d9-4437-ab19-3ae8eb202553'
+user_id= '6af522d0-2915-4e52-83a3-3690455a5fe6'
+
+def upgrade():
+    op.get_bind()
+    
+    # modify name of default service user in services
+    table_name = 'services'
+    col = 'name'
+    val = 'US Notify'
+    select_by_col = 'id'
+    select_by_val = service_id
+    op.execute(f"update {table_name} set {col}='{val}' where {select_by_col} = '{select_by_val}'")
+    
+    table_name = 'services_history'
+    op.execute(f"update {table_name} set {col}='{val}' where {select_by_col} = '{select_by_val}'")
+    
+
+def downgrade():
+    ### commands auto generated by Alembic - please adjust! ###
+    pass
+    ### end Alembic commands ###

migrations/versions/0376_add_provider_response.py

@@ -0,0 +1,30 @@
+"""empty message
+
+Revision ID: 0376_add_provider_response
+Revises: 0375_fix_service_name
+Create Date: 2022-09-14 11:04:15.888017
+
+"""
+# revision identifiers, used by Alembic.
+from datetime import datetime
+
+revision = '0376_add_provider_response'
+down_revision = '0375_fix_service_name'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+    ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('notifications', sa.Column('provider_response', sa.Text(), nullable=True))
+    op.add_column('notifications', sa.Column('queue_name', sa.Text(), nullable=True))
+    ### end Alembic commands ###
+
+
+def downgrade():
+    ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('notifications', 'provider_response')
+    op.drop_column('notifications', 'queue_name')
+    ### end Alembic commands ###
+    

requirements.in

@@ -6,7 +6,7 @@ celery[sqs]==5.2.6
 Flask-Bcrypt==1.0.1
 flask-marshmallow==0.14.0
 Flask-Migrate==3.1.0
-git+https://github.com/mitsuhiko/flask-sqlalchemy.git@500e732dd1b975a56ab06a46bd1a20a21e682262#egg=Flask-SQLAlchemy==2.3.2.dev20190108
+git+https://github.com/pallets-eco/flask-sqlalchemy.git@aa7a61a5357cf6f5dcc135d98c781192457aa6fa#egg=Flask-SQLAlchemy==2.5.1
 Flask==2.1.2
 click-datetime==0.2
 # Should be pinned until a new gunicorn release greater than 20.1.0 comes out. (Due to eventlet v0.33 compatibility issues)
@@ -14,23 +14,25 @@ git+https://github.com/benoitc/gunicorn.git@1299ea9e967a61ae2edebe191082fd169b86
 iso8601==1.0.2
 itsdangerous==2.1.2
 jsonschema[format]==4.5.1
-marshmallow-sqlalchemy==0.28.0
+marshmallow-sqlalchemy==0.28.1
 marshmallow==3.15.0
 psycopg2-binary==2.9.3
 PyJWT==2.4.0
-SQLAlchemy==1.4.36
+SQLAlchemy==1.4.40
 cachetools==5.1.0
 beautifulsoup4==4.11.1
 lxml==4.9.1
-Werkzeug==2.0.3  # pyup: <2.1.0 # later versions are not compatible with the version of flask-sqlalchemy we have pinned
+defusedxml==0.7.1
+Werkzeug==2.1.1
 python-dotenv==0.20.0
+oscrypto
 
 notifications-python-client==6.3.0
 
 # PaaS
 awscli-cwlogs==1.4.6
 
-notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@56.0.0
+notifications-utils @ git+https://github.com/GSA/notifications-utils.git
 
 # gds-metrics requires prometheseus 0.2.0, override that requirement as 0.7.1 brings significant performance gains
 prometheus-client==0.14.1

requirements.txt

@@ -10,6 +10,8 @@ amqp==5.1.1
     # via kombu
 arrow==1.2.2
     # via isoduration
+asn1crypto==1.5.1
+    # via oscrypto
 async-timeout==4.0.2
     # via redis
 attrs==21.4.0
@@ -69,6 +71,8 @@ click-repl==0.2.0
     # via celery
 colorama==0.4.4
     # via awscli
+defusedxml==0.7.1
+    # via -r requirements.in
 deprecated==1.2.13
     # via redis
 dnspython==2.2.1
@@ -86,6 +90,7 @@ flask==2.1.2
     #   flask-marshmallow
     #   flask-migrate
     #   flask-redis
+    #   flask-sqlalchemy
     #   gds-metrics
     #   notifications-utils
 flask-bcrypt==1.0.1
@@ -96,7 +101,7 @@ flask-migrate==3.1.0
     # via -r requirements.in
 flask-redis==0.4.0
     # via notifications-utils
-flask-sqlalchemy @ git+https://github.com/mitsuhiko/flask-sqlalchemy.git@500e732dd1b975a56ab06a46bd1a20a21e682262
+flask-sqlalchemy @ git+https://github.com/pallets-eco/flask-sqlalchemy.git@aa7a61a5357cf6f5dcc135d98c781192457aa6fa
     # via
     #   -r requirements.in
     #   flask-migrate
@@ -118,6 +123,8 @@ idna==3.3
     # via
     #   jsonschema
     #   requests
+importlib-metadata==4.12.0
+    # via flask
 iso8601==1.0.2
     # via -r requirements.in
 isoduration==20.11.0
@@ -154,20 +161,23 @@ marshmallow==3.15.0
     #   -r requirements.in
     #   flask-marshmallow
     #   marshmallow-sqlalchemy
-marshmallow-sqlalchemy==0.28.0
+marshmallow-sqlalchemy==0.28.1
     # via -r requirements.in
 mistune==0.8.4
     # via notifications-utils
 notifications-python-client==6.3.0
     # via -r requirements.in
-notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@56.0.0
+notifications-utils @ git+https://github.com/GSA/notifications-utils.git
     # via -r requirements.in
 orderedset==2.0.3
     # via notifications-utils
+oscrypto==1.3.0
+    # via -r requirements.in
 packaging==21.3
     # via
     #   bleach
     #   marshmallow
+    #   marshmallow-sqlalchemy
     #   redis
 phonenumbers==8.12.48
     # via notifications-utils
@@ -245,13 +255,16 @@ smartypants==2.0.1
     # via notifications-utils
 soupsieve==2.3.2.post1
     # via beautifulsoup4
-sqlalchemy==1.4.36
+sqlalchemy==1.4.40
     # via
     #   -r requirements.in
     #   alembic
+    #   flask-sqlalchemy
     #   marshmallow-sqlalchemy
 statsd==3.3.0
     # via notifications-utils
+typing-extensions==4.3.0
+    # via pypdf2
 uri-template==1.2.0
     # via jsonschema
 urllib3==1.26.9
@@ -269,12 +282,14 @@ webcolors==1.12
     # via jsonschema
 webencodings==0.5.1
     # via bleach
-werkzeug==2.0.3
+werkzeug==2.1.1
     # via
     #   -r requirements.in
     #   flask
 wrapt==1.14.1
     # via deprecated
+zipp==3.8.1
+    # via importlib-metadata
 
 # The following packages are considered to be unsafe in a requirements file:
 # setuptools

requirements_for_test.txt

@@ -1,4 +1,4 @@
--r requirements.txt
+--requirement requirements.txt
 flake8==4.0.1
 flake8-bugbear==22.4.25
 isort==5.10.1

sample.env

@@ -4,7 +4,7 @@
 DEBUG=True
 ANTIVIRUS_ENABLED=0
 NOTIFY_ENVIRONMENT=development
-NOTIFICATION_QUEUE_PREFIX=local_dev_10x
+NOTIFICATION_QUEUE_PREFIX=local_dev_YOURNAME_
 STATSD_HOST=localhost
 SES_STUB_URL=None
 NOTIFY_APP_NAME=api
@@ -33,11 +33,14 @@ API_HOST_NAME=http://dev:6011
 REDIS_URL=redis://redis:6380
 REDIS_ENABLED=1
 
-# DB conection string for local docker, overriden on remote with vcap env vars
+# DB connection string for local docker, overriden on remote with vcap env vars
 SQLALCHEMY_DATABASE_URI=postgresql://postgres:chummy@db:5432/notification_api
 
-# For GH actions CI workflow
+# For testing in local docker
-SQLALCHEMY_DATABASE_TEST_URI=postgresql://user:password@localhost:5432/test_notification_api
+SQLALCHEMY_DATABASE_TEST_URI=postgresql://postgres:chummy@db:5432/test_notification_api
+
+# DB connection string for local non-docker connection
+# SQLALCHEMY_DATABASE_URI=postgresql://user:password@localhost:5432/notification_api
 
 # AWS
 AWS_REGION=us-west-2

tests/app/celery/test_process_ses_receipts_tasks.py

@@ -10,12 +10,16 @@ from app.celery.research_mode_tasks import (
     ses_notification_callback,
     ses_soft_bounce_callback,
 )
+from app.celery.service_callback_tasks import (
+    create_delivery_status_callback_data,
+)
 from app.dao.notifications_dao import get_notification_by_id
 from app.models import Complaint, Notification
 from app.notifications.notifications_ses_callback import (
     remove_emails_from_bounce,
     remove_emails_from_complaint,
 )
+from tests.app.conftest import create_sample_notification
 from tests.app.db import (
     create_notification,
     create_service_callback_api,
@@ -23,16 +27,87 @@ from tests.app.db import (
 )
 
 
+def test_notifications_ses_400_with_invalid_header(client):
+    data = json.dumps({"foo": "bar"})
+    response = client.post(
+        path='/notifications/email/ses',
+        data=data,
+        headers=[('Content-Type', 'application/json')]
+    )
+    assert response.status_code == 400
+
+
+def test_notifications_ses_400_with_invalid_message_type(client):
+    data = json.dumps({"foo": "bar"})
+    response = client.post(
+        path='/notifications/email/ses',
+        data=data,
+        headers=[('Content-Type', 'application/json'), ('x-amz-sns-message-type', 'foo')]
+    )
+    assert response.status_code == 400
+    assert "SES-SNS callback failed: invalid message type" in response.get_data(as_text=True)
+
+
+def test_notifications_ses_400_with_invalid_json(client):
+    data = "FOOO"
+    response = client.post(
+        path='/notifications/email/ses',
+        data=data,
+        headers=[('Content-Type', 'application/json'), ('x-amz-sns-message-type', 'Notification')]
+    )
+    assert response.status_code == 400
+    assert "SES-SNS callback failed: invalid JSON given" in response.get_data(as_text=True)
+
+
+def test_notifications_ses_400_with_certificate(client):
+    data = json.dumps({"foo": "bar"})
+    response = client.post(
+        path='/notifications/email/ses',
+        data=data,
+        headers=[('Content-Type', 'application/json'), ('x-amz-sns-message-type', 'Notification')]
+    )
+    assert response.status_code == 400
+    assert "SES-SNS callback failed: validation failed" in response.get_data(as_text=True)
+
+
+def test_notifications_ses_200_autoconfirms_subscription(client, mocker):
+    mocker.patch("app.celery.process_ses_receipts_tasks.valid_sns_message", return_value=True)
+    requests_mock = mocker.patch("requests.get")
+    data = json.dumps({"Type": "SubscriptionConfirmation", "SubscribeURL": "https://foo"})
+    response = client.post(
+        path='/notifications/email/ses',
+        data=data,
+        headers=[('Content-Type', 'application/json'), ('x-amz-sns-message-type', 'SubscriptionConfirmation')]
+    )
+
+    requests_mock.assert_called_once_with("https://foo")
+    assert response.status_code == 200
+
+
+def test_notifications_ses_200_call_process_task(client, mocker):
+    mocker.patch("app.celery.process_ses_receipts_tasks.valid_sns_message", return_value=True)
+    process_mock = mocker.patch("app.celery.process_ses_receipts_tasks.process_ses_results.apply_async")
+    data = {"Type": "Notification", "foo": "bar"}
+    json_data = json.dumps(data)
+    response = client.post(
+        path='/notifications/email/ses',
+        data=json_data,
+        headers=[('Content-Type', 'application/json'), ('x-amz-sns-message-type', 'Notification')]
+    )
+
+    process_mock.assert_called_once_with([{'Message': None}], queue='notify-internal-tasks')
+    assert response.status_code == 200
+
+
 def test_process_ses_results(sample_email_template):
     create_notification(sample_email_template, reference='ref1', sent_at=datetime.utcnow(), status='sending')
 
     assert process_ses_results(response=ses_notification_callback(reference='ref1'))
 
 
-def test_process_ses_results_retry_called(sample_email_template, mocker):
+def test_process_ses_results_retry_called(sample_email_template, _notify_db, mocker):
     create_notification(sample_email_template, reference='ref1', sent_at=datetime.utcnow(), status='sending')
-
+    mocker.patch("app.dao.notifications_dao._update_notification_status", side_effect=Exception("EXPECTED"))
-    mocker.patch("app.dao.notifications_dao.dao_update_notifications_by_reference", side_effect=Exception("EXPECTED"))
     mocked = mocker.patch('app.celery.process_ses_receipts_tasks.process_ses_results.retry')
     process_ses_results(response=ses_notification_callback(reference='ref1'))
     assert mocked.call_count != 0
@@ -62,6 +137,7 @@ def test_remove_email_from_bounce():
 
 def test_ses_callback_should_update_notification_status(
         client,
+        _notify_db,
         notify_db_session,
         sample_email_template,
         mocker):
@@ -69,140 +145,159 @@ def test_ses_callback_should_update_notification_status(
         mocker.patch('app.statsd_client.incr')
         mocker.patch('app.statsd_client.timing_with_dates')
         send_mock = mocker.patch(
-            'app.celery.process_ses_receipts_tasks.check_and_queue_callback_task'
+            'app.celery.service_callback_tasks.send_delivery_status_to_service.apply_async'
         )
-        notification = create_notification(
+        notification = create_sample_notification(
+            _notify_db,
+            notify_db_session,
             template=sample_email_template,
-            status='sending',
             reference='ref',
+            status='sending',
+            sent_at=datetime.utcnow()
         )
+        callback_api = create_service_callback_api(service=sample_email_template.service, url="https://original_url.com")
         assert get_notification_by_id(notification.id).status == 'sending'
-
         assert process_ses_results(ses_notification_callback(reference='ref'))
         assert get_notification_by_id(notification.id).status == 'delivered'
         statsd_client.timing_with_dates.assert_any_call(
-            "callback.ses.delivered.elapsed-time", datetime.utcnow(), notification.sent_at
+            "callback.ses.elapsed-time", datetime.utcnow(), notification.sent_at
         )
         statsd_client.incr.assert_any_call("callback.ses.delivered")
         updated_notification = Notification.query.get(notification.id)
-        send_mock.assert_called_once_with(updated_notification)
+        encrypted_data = create_delivery_status_callback_data(updated_notification, callback_api)
+        send_mock.assert_called_once_with([str(notification.id), encrypted_data], queue="service-callbacks")
 
 
 def test_ses_callback_should_not_update_notification_status_if_already_delivered(sample_email_template, mocker):
     mock_dup = mocker.patch('app.celery.process_ses_receipts_tasks.notifications_dao._duplicate_update_warning')
-    mock_upd = mocker.patch(
+    mock_upd = mocker.patch('app.celery.process_ses_receipts_tasks.notifications_dao._update_notification_status')
-        'app.celery.process_ses_receipts_tasks.notifications_dao.dao_update_notifications_by_reference'
-    )
     notification = create_notification(template=sample_email_template, reference='ref', status='delivered')
-
     assert process_ses_results(ses_notification_callback(reference='ref')) is None
     assert get_notification_by_id(notification.id).status == 'delivered'
-
+    mock_dup.assert_called_once_with(notification, 'delivered')
-    mock_dup.assert_called_once_with(notification=notification, status='delivered')
     assert mock_upd.call_count == 0
 
 
-def test_ses_callback_should_retry_if_notification_is_new(client, notify_db_session, mocker):
+def test_ses_callback_should_retry_if_notification_is_new(client, _notify_db, mocker):
     mock_retry = mocker.patch('app.celery.process_ses_receipts_tasks.process_ses_results.retry')
     mock_logger = mocker.patch('app.celery.process_ses_receipts_tasks.current_app.logger.error')
-
     with freeze_time('2017-11-17T12:14:03.646Z'):
         assert process_ses_results(ses_notification_callback(reference='ref')) is None
         assert mock_logger.call_count == 0
         assert mock_retry.call_count == 1
-
+def test_ses_callback_should_log_if_notification_is_missing(client, _notify_db, mocker):
-
-def test_ses_callback_should_log_if_notification_is_missing(client, notify_db_session, mocker):
     mock_retry = mocker.patch('app.celery.process_ses_receipts_tasks.process_ses_results.retry')
     mock_logger = mocker.patch('app.celery.process_ses_receipts_tasks.current_app.logger.warning')
-
     with freeze_time('2017-11-17T12:34:03.646Z'):
         assert process_ses_results(ses_notification_callback(reference='ref')) is None
         assert mock_retry.call_count == 0
-        mock_logger.assert_called_once_with('notification not found for reference: ref (update to delivered)')
+        mock_logger.assert_called_once_with('notification not found for reference: ref (while attempting update to delivered)')
-
+def test_ses_callback_should_not_retry_if_notification_is_old(client, _notify_db, mocker):
-
-def test_ses_callback_should_not_retry_if_notification_is_old(client, notify_db_session, mocker):
     mock_retry = mocker.patch('app.celery.process_ses_receipts_tasks.process_ses_results.retry')
     mock_logger = mocker.patch('app.celery.process_ses_receipts_tasks.current_app.logger.error')
-
     with freeze_time('2017-11-21T12:14:03.646Z'):
         assert process_ses_results(ses_notification_callback(reference='ref')) is None
         assert mock_logger.call_count == 0
         assert mock_retry.call_count == 0
-
+def test_ses_callback_does_not_call_send_delivery_status_if_no_db_entry(
-
-def test_ses_callback_should_update_multiple_notification_status_sent(
         client,
+        _notify_db,
+        notify_db_session,
+        sample_email_template,
+        mocker):
+    with freeze_time('2001-01-01T12:00:00'):
+        send_mock = mocker.patch(
+            'app.celery.service_callback_tasks.send_delivery_status_to_service.apply_async'
+        )
+        notification = create_sample_notification(
+            _notify_db,
+            notify_db_session,
+            template=sample_email_template,
+            reference='ref',
+            status='sending',
+            sent_at=datetime.utcnow()
+        )
+        assert get_notification_by_id(notification.id).status == 'sending'
+        assert process_ses_results(ses_notification_callback(reference='ref'))
+        assert get_notification_by_id(notification.id).status == 'delivered'
+        send_mock.assert_not_called()
+def test_ses_callback_should_update_multiple_notification_status_sent(
+        client,
+        _notify_db,
         notify_db_session,
         sample_email_template,
         mocker):
-
     send_mock = mocker.patch(
-        'app.celery.process_ses_receipts_tasks.check_and_queue_callback_task'
+        'app.celery.service_callback_tasks.send_delivery_status_to_service.apply_async'
     )
-    create_notification(
+    create_sample_notification(
+        _notify_db,
+        notify_db_session,
         template=sample_email_template,
-        status='sending',
         reference='ref1',
-    )
+        sent_at=datetime.utcnow(),
-    create_notification(
+        status='sending')
+    create_sample_notification(
+        _notify_db,
+        notify_db_session,
         template=sample_email_template,
-        status='sending',
         reference='ref2',
-    )
+        sent_at=datetime.utcnow(),
-    create_notification(
+        status='sending')
+    create_sample_notification(
+        _notify_db,
+        notify_db_session,
         template=sample_email_template,
-        status='sending',
         reference='ref3',
-    )
+        sent_at=datetime.utcnow(),
+        status='sending')
+    create_service_callback_api(service=sample_email_template.service, url="https://original_url.com")
     assert process_ses_results(ses_notification_callback(reference='ref1'))
     assert process_ses_results(ses_notification_callback(reference='ref2'))
     assert process_ses_results(ses_notification_callback(reference='ref3'))
     assert send_mock.called
-
-
 def test_ses_callback_should_set_status_to_temporary_failure(client,
+                                                             _notify_db,
                                                              notify_db_session,
                                                              sample_email_template,
                                                              mocker):
     send_mock = mocker.patch(
-        'app.celery.process_ses_receipts_tasks.check_and_queue_callback_task'
+        'app.celery.service_callback_tasks.send_delivery_status_to_service.apply_async'
     )
-    mock_logger = mocker.patch('app.celery.process_ses_receipts_tasks.current_app.logger.info')
+    notification = create_sample_notification(
-    notification = create_notification(
+        _notify_db,
+        notify_db_session,
         template=sample_email_template,
-        status='sending',
         reference='ref',
+        status='sending',
+        sent_at=datetime.utcnow()
     )
+    create_service_callback_api(service=notification.service, url="https://original_url.com")
     assert get_notification_by_id(notification.id).status == 'sending'
     assert process_ses_results(ses_soft_bounce_callback(reference='ref'))
     assert get_notification_by_id(notification.id).status == 'temporary-failure'
     assert send_mock.called
-    assert f'SES bounce for notification ID {notification.id}: ' in mock_logger.call_args[0][0]
-
-
 def test_ses_callback_should_set_status_to_permanent_failure(client,
+                                                             _notify_db,
                                                              notify_db_session,
                                                              sample_email_template,
                                                              mocker):
     send_mock = mocker.patch(
-        'app.celery.process_ses_receipts_tasks.check_and_queue_callback_task'
+        'app.celery.service_callback_tasks.send_delivery_status_to_service.apply_async'
     )
-    mock_logger = mocker.patch('app.celery.process_ses_receipts_tasks.current_app.logger.info')
+    notification = create_sample_notification(
-    notification = create_notification(
+        _notify_db,
+        notify_db_session,
         template=sample_email_template,
-        status='sending',
         reference='ref',
+        status='sending',
+        sent_at=datetime.utcnow()
     )
-
+    create_service_callback_api(service=sample_email_template.service, url="https://original_url.com")
     assert get_notification_by_id(notification.id).status == 'sending'
     assert process_ses_results(ses_hard_bounce_callback(reference='ref'))
     assert get_notification_by_id(notification.id).status == 'permanent-failure'
     assert send_mock.called
-    assert f'SES bounce for notification ID {notification.id}: ' in mock_logger.call_args[0][0]
-
-
 def test_ses_callback_should_send_on_complaint_to_user_callback_api(sample_email_template, mocker):
     send_mock = mocker.patch(
         'app.celery.service_callback_tasks.send_complaint_to_service.apply_async'
@@ -210,13 +305,11 @@ def test_ses_callback_should_send_on_complaint_to_user_callback_api(sample_email
     create_service_callback_api(
         service=sample_email_template.service, url="https://original_url.com", callback_type="complaint"
     )
-
     notification = create_notification(
         template=sample_email_template, reference='ref1', sent_at=datetime.utcnow(), status='sending'
     )
     response = ses_complaint_callback()
     assert process_ses_results(response)
-
     assert send_mock.call_count == 1
     assert encryption.decrypt(send_mock.call_args[0][0][0]) == {
         'complaint_date': '2018-06-05T13:59:58.000000Z',
@@ -227,3 +320,4 @@ def test_ses_callback_should_send_on_complaint_to_user_callback_api(sample_email
         'service_callback_api_url': 'https://original_url.com',
         'to': 'recipient1@example.com'
     }
+    

tests/app/conftest.py

@@ -31,6 +31,7 @@ from app.models import (
     KEY_TYPE_TEAM,
     KEY_TYPE_TEST,
     LETTER_TYPE,
+    NOTIFICATION_STATUS_TYPES_COMPLETED,
     SERVICE_PERMISSION_TYPES,
     SMS_TYPE,
     ApiKey,
@@ -69,6 +70,86 @@ def rmock():
         yield rmock
 
 
+def create_sample_notification(
+    notify_db,
+    notify_db_session,
+    service=None,
+    template=None,
+    job=None,
+    job_row_number=None,
+    to_field=None,
+    status="created",
+    provider_response=None,
+    reference=None,
+    created_at=None,
+    sent_at=None,
+    billable_units=1,
+    personalisation=None,
+    api_key=None,
+    key_type=KEY_TYPE_NORMAL,
+    sent_by=None,
+    international=False,
+    client_reference=None,
+    rate_multiplier=1.0,
+    scheduled_for=None,
+    normalised_to=None,
+    postage=None,
+):
+    if created_at is None:
+        created_at = datetime.utcnow()
+    if service is None:
+        service = create_service(check_if_service_exists=True)
+    if template is None:
+        template = create_template(service=service)
+
+    if job is None and api_key is None:
+        # we didn't specify in test - lets create it
+        api_key = ApiKey.query.filter(ApiKey.service == template.service, ApiKey.key_type == key_type).first()
+        if not api_key:
+            api_key = create_api_key(template.service, key_type=key_type)
+
+    notification_id = uuid.uuid4()
+
+    if to_field:
+        to = to_field
+    else:
+        to = "+16502532222"
+
+    data = {
+        "id": notification_id,
+        "to": to,
+        "job_id": job.id if job else None,
+        "job": job,
+        "service_id": service.id,
+        "service": service,
+        "template_id": template.id,
+        "template_version": template.version,
+        "status": status,
+        "provider_response": provider_response,
+        "reference": reference,
+        "created_at": created_at,
+        "sent_at": sent_at,
+        "billable_units": billable_units,
+        "personalisation": personalisation,
+        "notification_type": template.template_type,
+        "api_key": api_key,
+        "api_key_id": api_key and api_key.id,
+        "key_type": api_key.key_type if api_key else key_type,
+        "sent_by": sent_by,
+        "updated_at": created_at if status in NOTIFICATION_STATUS_TYPES_COMPLETED else None,
+        "client_reference": client_reference,
+        "rate_multiplier": rate_multiplier,
+        "normalised_to": normalised_to,
+        "postage": postage,
+    }
+    if job_row_number is not None:
+        data["job_row_number"] = job_row_number
+    notification = Notification(**data)
+    dao_create_notification(notification)
+
+    return notification
+
+
 @pytest.fixture(scope='function')
 def service_factory(sample_user):
     class ServiceFactory(object):

tests/app/dao/notification_dao/test_notification_dao.py

@@ -15,7 +15,7 @@ from app.dao.notifications_dao import (
     dao_get_letters_to_be_printed,
     dao_get_notification_by_reference,
     dao_get_notification_count_for_job_id,
-    dao_get_notification_or_history_by_reference,
+    dao_get_notification_history_by_reference,
     dao_get_notifications_by_recipient_or_reference,
     dao_timeout_notifications,
     dao_update_notification,
@@ -1607,28 +1607,28 @@ def test_dao_get_notification_by_reference_with_no_matches_raises_error(notify_d
         dao_get_notification_by_reference('REF1')
 
 
-def test_dao_get_notification_or_history_by_reference_with_one_match_returns_notification(
+def test_dao_get_notification_history_by_reference_with_one_match_returns_notification(
         sample_letter_template
 ):
     create_notification(template=sample_letter_template, reference='REF1')
-    notification = dao_get_notification_or_history_by_reference('REF1')
+    notification = dao_get_notification_history_by_reference('REF1')
 
     assert notification.reference == 'REF1'
 
 
-def test_dao_get_notification_or_history_by_reference_with_multiple_matches_raises_error(
+def test_dao_get_notification_history_by_reference_with_multiple_matches_raises_error(
         sample_letter_template
 ):
     create_notification(template=sample_letter_template, reference='REF1')
     create_notification(template=sample_letter_template, reference='REF1')
 
     with pytest.raises(SQLAlchemyError):
-        dao_get_notification_or_history_by_reference('REF1')
+        dao_get_notification_history_by_reference('REF1')
 
 
-def test_dao_get_notification_or_history_by_reference_with_no_matches_raises_error(notify_db_session):
+def test_dao_get_notification_history_by_reference_with_no_matches_raises_error(notify_db_session):
     with pytest.raises(SQLAlchemyError):
-        dao_get_notification_or_history_by_reference('REF1')
+        dao_get_notification_history_by_reference('REF1')
 
 
 @pytest.mark.parametrize("notification_type",

tests/app/v2/notifications/test_get_notifications.py

@@ -68,6 +68,7 @@ def test_get_notification_by_id_returns_200(
         'completed_at': sample_notification.completed_at(),
         'scheduled_for': None,
         'postage': None,
+        'provider_response': None
     }
 
     assert json_response == expected_response
@@ -120,6 +121,7 @@ def test_get_notification_by_id_with_placeholders_returns_200(
         'completed_at': sample_notification.completed_at(),
         'scheduled_for': None,
         'postage': None,
+        'provider_response': None
     }
 
     assert json_response == expected_response

zap.conf

@@ -0,0 +1,121 @@
+# zap-full-scan rule configuration file
+# Change WARN to IGNORE to ignore rule or FAIL to fail if rule matches
+# Active scan rules set to IGNORE will not be run which will speed up the scan
+# Only the rule identifiers are used - the names are just for info
+# You can add your own messages to each rule by appending them after a tab on each line.
+0	WARN	(Directory Browsing - Active/release)
+10003	WARN	(Vulnerable JS Library - Passive/release)
+10010	FAIL	(Cookie No HttpOnly Flag - Passive/release)
+10011	FAIL	(Cookie Without Secure Flag - Passive/release)
+10015	WARN	(Incomplete or No Cache-control Header Set - Passive/release)
+10016	FAIL	(Web Browser XSS Protection Not Enabled)
+10017	WARN	(Cross-Domain JavaScript Source File Inclusion - Passive/release)
+10019	WARN	(Content-Type Header Missing - Passive/release)
+10020	FAIL	(X-Frame-Options Header - Passive/release)
+10021	WARN	(X-Content-Type-Options Header Missing - Passive/release)
+10023	WARN	(Information Disclosure - Debug Error Messages - Passive/release)
+10024	FAIL	(Information Disclosure - Sensitive Information in URL - Passive/release)
+10025	FAIL	(Information Disclosure - Sensitive Information in HTTP Referrer Header - Passive/release)
+10026	WARN	(HTTP Parameter Override - Passive/beta)
+10027	WARN	(Information Disclosure - Suspicious Comments - Passive/release)
+10028	FAIL	(Open Redirect - Passive/beta)
+10029	WARN	(Cookie Poisoning - Passive/beta)
+10030	WARN	(User Controllable Charset - Passive/beta)
+10031	WARN	(User Controllable HTML Element Attribute (Potential XSS) - Passive/beta)
+10032	WARN	(Viewstate - Passive/release)
+10033	WARN	(Directory Browsing - Passive/beta)
+10034	WARN	(Heartbleed OpenSSL Vulnerability (Indicative) - Passive/beta)
+10035	FAIL	(Strict-Transport-Security Header - Passive/beta)
+10036	WARN	(HTTP Server Response Header - Passive/beta)
+10037	WARN	(Server Leaks Information via "X-Powered-By" HTTP Response Header Field(s) - Passive/release)
+10038	FAIL	(Content Security Policy (CSP) Header Not Set - Passive/beta)
+10039	WARN	(X-Backend-Server Header Information Leak - Passive/beta)
+10040	FAIL	(Secure Pages Include Mixed Content - Passive/release)
+10041	WARN	(HTTP to HTTPS Insecure Transition in Form Post - Passive/beta)
+10042	WARN	(HTTPS to HTTP Insecure Transition in Form Post - Passive/beta)
+10043	FAIL	(User Controllable JavaScript Event (XSS) - Passive/beta)
+10044	WARN	(Big Redirect Detected (Potential Sensitive Information Leak) - Passive/beta)
+10045	WARN	(Source Code Disclosure - /WEB-INF folder - Active/release)
+10047	WARN	(HTTPS Content Available via HTTP - Active/beta)
+10048	FAIL	(Remote Code Execution - Shell Shock - Active/beta)
+10050	WARN	(Retrieved from Cache - Passive/beta)
+10051	WARN	(Relative Path Confusion - Active/beta)
+10052	WARN	(X-ChromeLogger-Data (XCOLD) Header Information Leak - Passive/beta)
+10053	WARN	(Apache Range Header DoS (CVE-2011-3192) - Active/beta)
+10054	WARN	(Cookie without SameSite Attribute - Passive/release)
+10055	WARN	(CSP - Passive/release)
+10056	WARN	(X-Debug-Token Information Leak - Passive/release)
+10057	WARN	(Username Hash Found - Passive/release)
+10058	FAIL	(GET for POST - Active/beta)
+10061	WARN	(X-AspNet-Version Response Header - Passive/release)
+10062	FAIL	(PII Disclosure - Passive/beta)
+10095	IGNORE	(Backup File Disclosure - Active/beta)
+10096	WARN	(Timestamp Disclosure - Passive/release)
+10097	WARN	(Hash Disclosure - Passive/beta)
+10098	WARN	(Cross-Domain Misconfiguration - Passive/release)
+10104	WARN	(User Agent Fuzzer - Active/beta)
+10105	WARN	(Weak Authentication Method - Passive/release)
+10106	IGNORE	(HTTP Only Site - Active/beta)
+10107	WARN	(Httpoxy - Proxy Header Misuse - Active/beta)
+10108	WARN	(Reverse Tabnabbing - Passive/beta)
+10109	WARN	(Modern Web Application - Passive/beta)
+10202	FAIL	(Absence of Anti-CSRF Tokens - Passive/release)
+2	WARN	(Private IP Disclosure - Passive/release)
+20012	FAIL	(Anti-CSRF Tokens Check - Active/beta)
+20014	WARN	(HTTP Parameter Pollution - Active/beta)
+20015	WARN	(Heartbleed OpenSSL Vulnerability - Active/beta)
+20016	WARN	(Cross-Domain Misconfiguration - Active/beta)
+20017	FAIL	(Source Code Disclosure - CVE-2012-1823 - Active/beta)
+20018	FAIL	(Remote Code Execution - CVE-2012-1823 - Active/beta)
+20019	WARN	(External Redirect - Active/release)
+3	WARN	(Session ID in URL Rewrite - Passive/release)
+30001	WARN	(Buffer Overflow - Active/release)
+30002	WARN	(Format String Error - Active/release)
+30003	WARN	(Integer Overflow Error - Active/beta)
+40003	WARN	(CRLF Injection - Active/release)
+40008	WARN	(Parameter Tampering - Active/release)
+40009	WARN	(Server Side Include - Active/release)
+40012	FAIL	(Cross Site Scripting (Reflected) - Active/release)
+40013	FAIL	(Session Fixation - Active/beta)
+40014	FAIL	(Cross Site Scripting (Persistent) - Active/release)
+40016	FAIL	(Cross Site Scripting (Persistent) - Prime - Active/release)
+40017	FAIL	(Cross Site Scripting (Persistent) - Spider - Active/release)
+40018	FAIL	(SQL Injection - Active/release)
+40019	FAIL	(SQL Injection - MySQL - Active/beta)
+40020	FAIL	(SQL Injection - Hypersonic SQL - Active/beta)
+40021	FAIL	(SQL Injection - Oracle - Active/beta)
+40022	FAIL	(SQL Injection - PostgreSQL - Active/beta)
+40023	FAIL	(Possible Username Enumeration - Active/beta)
+40024	FAIL	(SQL Injection - SQLite - Active/beta)
+40025	FAIL	(Proxy Disclosure - Active/beta)
+40026	FAIL	(Cross Site Scripting (DOM Based) - Active/beta)
+40027	FAIL	(SQL Injection - MsSQL - Active/beta)
+40028	WARN	(ELMAH Information Leak - Active/release)
+40029	WARN	(Trace.axd Information Leak - Active/beta)
+40032	FAIL	(.htaccess Information Leak - Active/release)
+40034	FAIL	(.env Information Leak - Active/beta)
+40035	FAIL	(Hidden File Finder - Active/beta)
+41	FAIL	(Source Code Disclosure - Git  - Active/beta)
+42	WARN	(Source Code Disclosure - SVN - Active/beta)
+43	WARN	(Source Code Disclosure - File Inclusion - Active/beta)
+50000	WARN	(Script Active Scan Rules - Active/release)
+50001	WARN	(Script Passive Scan Rules - Passive/release)
+6	WARN	(Path Traversal - Active/release)
+7	WARN	(Remote File Inclusion - Active/release)
+90001	WARN	(Insecure JSF ViewState - Passive/release)
+90011	WARN	(Charset Mismatch - Passive/release)
+90017	WARN	(XSLT Injection - Active/beta)
+90019	WARN	(Server Side Code Injection - Active/release)
+90020	FAIL	(Remote OS Command Injection - Active/release)
+90021	WARN	(XPath Injection - Active/beta)
+90022	WARN	(Application Error Disclosure - Passive/release)
+90023	WARN	(XML External Entity Attack - Active/beta)
+90024	WARN	(Generic Padding Oracle - Active/beta)
+90025	WARN	(Expression Language Injection - Active/beta)
+90026	WARN	(SOAP Action Spoofing - Active/alpha)
+90027	IGNORE	(Cookie Slack Detector - Active/beta)
+90028	WARN	(Insecure HTTP Method - Active/beta)
+90029	WARN	(SOAP XML Injection - Active/alpha)
+90030	WARN	(WSDL File Detection - Passive/alpha)
+90033	WARN	(Loosely Scoped Cookie - Passive/release)
+90034	WARN	(Cloud Metadata Potentially Exposed - Active/beta)