Merge pull request #25 from 18F/security-scans

Security & Compliance scans
2026-07-10 03:14:19 -04:00 · 2022-08-29 13:34:43 -04:00
parent 55bc3e54ed f90f32f032
commit b1418b277e
20 changed files with 373 additions and 67 deletions
--- a/.github/actions/setup-project/action.yml
+++ b/.github/actions/setup-project/action.yml
@@ -0,0 +1,15 @@
+name: Set up project
+description: Setup python & install dependencies
+runs:
+  using: composite
+  steps:
+    - name: Install container dependencies
+      shell: bash
+      run: |
+        sudo apt-get update \
+        && sudo apt-get install -y --no-install-recommends \
+        libcurl4-openssl-dev
+    - name: Set up Python 3.9
+      uses: actions/setup-python@v3
+      with:
+        python-version: "3.9"
--- a/.github/workflows/checks.yml
+++ b/.github/workflows/checks.yml
@@ -50,25 +50,75 @@ jobs:
          - 5432:5432

    steps:
-    - name: Install container dependencies
-      run: |
-        sudo apt-get update \
-        && sudo apt-get install -y --no-install-recommends \
-        libcurl4-openssl-dev
    - uses: actions/checkout@v3
-    - name: Set up Python 3.9
-      uses: actions/setup-python@v3
-      with:
-        python-version: "3.9"
+    - uses: ./.github/actions/setup-project
    - name: Install application dependencies
      run: make bootstrap
      env:
        SQLALCHEMY_DATABASE_TEST_URI: postgresql://user:password@localhost:5432/test_notification_api
    # - name: Run style checks
    #   run: flake8 .
-    # - name: Check imports alphabetized
-    #   run: isort --check-only ./app ./tests
+    - name: Check imports alphabetized
+      run: isort --check-only ./app ./tests
    - name: Run tests
      run: pytest -n4 --maxfail=10
      env:
        SQLALCHEMY_DATABASE_TEST_URI: postgresql://user:password@localhost:5432/test_notification_api
+
+  pip-audit:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/setup-project
+      - uses: trailofbits/gh-action-pip-audit@v1.0.0
+        with:
+          inputs: requirements.txt requirements_for_test.txt
+          ignore-vulns: PYSEC-2022-237
+
+  static-scan:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/setup-project
+      - name: Install bandit
+        run: pip install bandit
+      - name: Run scan
+        run: bandit -r app/ --confidence-level medium
+
+  dynamic-scan:
+    runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_USER: user
+          POSTGRES_PASSWORD: password
+          POSTGRES_DB: test_notification_api
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          # Maps tcp port 5432 on service container to the host
+          - 5432:5432
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/setup-project
+      - name: Install application dependencies
+        run: make bootstrap
+        env:
+          SQLALCHEMY_DATABASE_TEST_URI: postgresql://user:password@localhost:5432/test_notification_api
+      - name: Run server
+        run: make run-flask &
+        env:
+          SQLALCHEMY_DATABASE_TEST_URI: postgresql://user:password@localhost:5432/test_notification_api
+      - name: Run OWASP Baseline Scan
+        uses: zaproxy/action-api-scan@v0.1.1
+        with:
+          docker_name: 'owasp/zap2docker-weekly'
+          target: 'http://localhost:6011/_status'
+          fail_action: true
+          allow_issue_writing: false
+          rules_file_name: 'zap.conf'
+          cmd_options: '-I'
--- a/.github/workflows/daily_checks.yml
+++ b/.github/workflows/daily_checks.yml
@@ -0,0 +1,93 @@
+name: Run daily scans
+
+on:
+  schedule:
+    # cron format: 'minute hour dayofmonth month dayofweek'
+    # this will run at noon UTC every day (7am EST / 8am EDT)
+    - cron: '0 12 * * *'
+
+permissions:
+  contents: read
+
+env:
+  DEBUG: True
+  ANTIVIRUS_ENABLED: 0
+  NOTIFY_ENVIRONMENT: test
+  NOTIFICATION_QUEUE_PREFIX: local_dev_10x
+  STATSD_HOST: localhost
+  SES_STUB_URL: None
+  NOTIFY_APP_NAME: api
+  NOTIFY_EMAIL_DOMAIN: dispostable.com
+  NOTIFY_LOG_PATH: /workspace/logs/app.log
+  ADMIN_CLIENT_ID: notify-admin
+  ADMIN_CLIENT_SECRET: dev-notify-secret-key
+  GOVUK_ALERTS_CLIENT_ID: govuk-alerts
+  FLASK_APP: application.py
+  FLASK_ENV: development
+  WERKZEUG_DEBUG_PIN: off
+  ADMIN_BASE_URL: http://localhost:6012
+  API_HOST_NAME: http://localhost:6011
+  REDIS_URL: redis://localhost:6380
+  REDIS_ENABLED: False
+  AWS_REGION: us-west-2
+  AWS_PINPOINT_REGION: us-west-2
+  AWS_US_TOLL_FREE_NUMBER: +18446120782
+
+jobs:
+  pip-audit:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/setup-project
+      - uses: trailofbits/gh-action-pip-audit@v1.0.0
+        with:
+          inputs: requirements.txt requirements_for_test.txt
+          ignore-vulns: PYSEC-2022-237
+
+  static-scan:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/setup-project
+      - name: Install bandit
+        run: pip install bandit
+      - name: Run scan
+        run: bandit -r app/ --confidence-level medium
+
+  dynamic-scan:
+    runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_USER: user
+          POSTGRES_PASSWORD: password
+          POSTGRES_DB: test_notification_api
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          # Maps tcp port 5432 on service container to the host
+          - 5432:5432
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/setup-project
+      - name: Install application dependencies
+        run: make bootstrap
+        env:
+          SQLALCHEMY_DATABASE_TEST_URI: postgresql://user:password@localhost:5432/test_notification_api
+      - name: Run server
+        run: make run-flask &
+        env:
+          SQLALCHEMY_DATABASE_TEST_URI: postgresql://user:password@localhost:5432/test_notification_api
+      - name: Run OWASP Baseline Scan
+        uses: zaproxy/action-api-scan@v0.1.1
+        with:
+          docker_name: 'owasp/zap2docker-weekly'
+          target: 'http://localhost:6011/_status'
+          fail_action: true
+          allow_issue_writing: false
+          rules_file_name: 'zap.conf'
+          cmd_options: '-I'
--- a/10
+++ b/10
@@ -77,6 +77,16 @@ freeze-requirements: ## Pin all requirements including sub dependencies into req
 	pip install --upgrade pip-tools
 	pip-compile requirements.in

+.PHONY: audit
+audit:
+	pip install --upgrade pip-audit
+	pip-audit -r requirements.txt -r requirements_for_test.txt -l --ignore-vuln PYSEC-2022-237
+
+.PHONY: static-scan
+static-scan:
+	pip install bandit
+	bandit -r app/
+
 .PHONY: clean
 clean:
 	rm -rf node_modules cache target venv .coverage build tests/.cache ${CF_MANIFEST_PATH}
--- a/README.md
+++ b/README.md
@@ -24,7 +24,7 @@ Create the external docker network:
 `docker network create notify-network`

 Using the command palette (shift+cmd+p), search and select “Remote Containers: Open Folder in Container...”
-When prompted, choose **devcontainer-api** folder (note: this is a *subfolder* of notification-api). This will startup the container in a new window (replacing the current one). 
+When prompted, choose **devcontainer-api** folder (note: this is a *subfolder* of notification-api). This will startup the container in a new window (replacing the current one).

 After this page loads, hit "show logs” in bottom-right. The first time this runs it will need to build the Docker image, which will likely take several minutes.

@@ -39,7 +39,7 @@ Open another terminal and run the background tasks:

 ### `.env` file

-Create and edit a .env file, based on sample.env. 
+Create and edit a .env file, based on sample.env.

 NOTE: when you change .env in the future, you'll need to rebuild the devcontainer for the change to take effect. Vscode _should_ detect the change and prompt you with a toast notification during a cached build. If not, you can find a manual rebuild in command pallette or just `docker rm` the notifications-api container.

@@ -109,10 +109,19 @@ make bootstrap
 make test
 ```

+## To run a local OWASP scan
+
+1. Run `make run-flask` from within the dev container.
+2. On your host machine run:
+
+```
+docker run -v $(pwd):/zap/wrk/:rw --network="notify-network" -t owasp/zap2docker-weekly zap-api-scan.py -t http://dev:6011/_status -f openapi -c zap.conf
+```
+
 ## To run scheduled tasks

 ```
-# After scheduling some tasks, open a third terminal in your running devcontainer and run celery beat 
+# After scheduling some tasks, open a third terminal in your running devcontainer and run celery beat
 make run-celery-beat
 ```

--- a/app/init.py
+++ b/app/init.py
@@ -1,5 +1,5 @@
 import os
-import random
+import secrets
 import string
 import time
 import uuid
@@ -41,12 +41,13 @@ class SQLAlchemy(_SQLAlchemy):
    """We need to subclass SQLAlchemy in order to override create_engine options"""

    def apply_driver_hacks(self, app, info, options):
-        super().apply_driver_hacks(app, info, options)
+        sa_url, options = super().apply_driver_hacks(app, info, options)
        if 'connect_args' not in options:
            options['connect_args'] = {}
        options['connect_args']["options"] = "-c statement_timeout={}".format(
            int(app.config['SQLALCHEMY_STATEMENT_TIMEOUT']) * 1000
        )
+        return (sa_url, options)


 db = SQLAlchemy()
@@ -352,7 +353,7 @@ def create_uuid():


 def create_random_identifier():
-    return ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(16))
+    return ''.join(secrets.choice(string.ascii_uppercase + string.digits) for _ in range(16))


 def setup_sqlalchemy_events(app):
--- a/app/authentication/auth.py
+++ b/app/authentication/auth.py
@@ -18,7 +18,7 @@ from sqlalchemy.orm.exc import NoResultFound

 from app.serialised_models import SerialisedService

-GENERAL_TOKEN_ERROR_MESSAGE = 'Invalid token: make sure your API token matches the example at https://docs.notifications.service.gov.uk/rest-api.html#authorisation-header'  # noqa
+GENERAL_TOKEN_ERROR_MESSAGE = 'Invalid token: make sure your API token matches the example at https://docs.notifications.service.gov.uk/rest-api.html#authorisation-header'  # nosec B105

 AUTH_DB_CONNECTION_DURATION_SECONDS = Histogram(
    'auth_db_connection_duration_seconds',
--- a/app/celery/process_sms_client_response_tasks.py
+++ b/app/celery/process_sms_client_response_tasks.py
@@ -14,13 +14,13 @@ from app.notifications.notifications_ses_callback import (
    check_and_queue_callback_task,
 )

-# sms_response_mapper = {
-#     'MMG': get_mmg_responses,
-#     'Firetext': get_firetext_responses,
-# }
+sms_response_mapper = {
+    # 'MMG': get_mmg_responses,
+    # 'Firetext': get_firetext_responses,
+}


-gUpdate with new providers")
+# gUpdate with new providers")
@notify_celery.task(bind=True, name="process-sms-client-response", max_retries=5, default_retry_delay=300)
 def process_sms_client_response(self, status, provider_reference, client_name, detailed_status_code=None):
    # validate reference
--- a/app/celery/research_mode_tasks.py
+++ b/app/celery/research_mode_tasks.py
@@ -124,7 +124,7 @@ def create_fake_letter_response_file(self, reference):
    dvla_response_data = '{}|Sent|0|Sorted'.format(reference)

    # try and find a filename that hasn't been taken yet - from a random time within the last 30 seconds
-    for i in sorted(range(30), key=lambda _: random.random()):
+    for i in sorted(range(30), key=lambda _: random.random()): # nosec B311 - not security related
        upload_file_name = 'NOTIFY-{}-RSP.TXT'.format((now - timedelta(seconds=i)).strftime('%Y%m%d%H%M%S'))
        if not file_exists(current_app.config['DVLA_RESPONSE_BUCKET_NAME'], upload_file_name):
            break
--- a/app/clients/init.py
+++ b/app/clients/init.py
@@ -38,8 +38,8 @@ class NotificationProviderClients(object):
        return self.email_clients.get(name)

    def get_client_by_name_and_type(self, name, notification_type):
-        assert notification_type in ['email', 'sms']
-        
+        assert notification_type in ['email', 'sms'] # nosec B101
+
        if notification_type == 'email':
            return self.get_email_client(name)

--- a/app/commands.py
+++ b/app/commands.py
@@ -151,8 +151,8 @@ def backfill_notification_statuses():
    `Notification._status_enum`
    """
    LIMIT = 250000
-    subq = "SELECT id FROM notification_history WHERE notification_status is NULL LIMIT {}".format(LIMIT)
-    update = "UPDATE notification_history SET notification_status = status WHERE id in ({})".format(subq)
+    subq = "SELECT id FROM notification_history WHERE notification_status is NULL LIMIT {}".format(LIMIT) # nosec B608 no user-controlled input
+    update = "UPDATE notification_history SET notification_status = status WHERE id in ({})".format(subq) # nosec B608 no user-controlled input
    result = db.session.execute(subq).fetchall()

    while len(result) > 0:
@@ -169,7 +169,7 @@ def update_notification_international_flag():
    """
    # 250,000 rows takes 30 seconds to update.
    subq = "select id from notifications where international is null limit 250000"
-    update = "update notifications set international = False where id in ({})".format(subq)
+    update = "update notifications set international = False where id in ({})".format(subq) # nosec B608 no user-controlled input
    result = db.session.execute(subq).fetchall()

    while len(result) > 0:
@@ -180,7 +180,7 @@ def update_notification_international_flag():

    # Now update notification_history
    subq_history = "select id from notification_history where international is null limit 250000"
-    update_history = "update notification_history set international = False where id in ({})".format(subq_history)
+    update_history = "update notification_history set international = False where id in ({})".format(subq_history) # nosec B608 no user-controlled input
    result_history = db.session.execute(subq_history).fetchall()
    while len(result_history) > 0:
        db.session.execute(update_history)
@@ -201,8 +201,8 @@ def fix_notification_statuses_not_in_sync():
    """
    MAX = 10000

-    subq = "SELECT id FROM notifications WHERE cast (status as text) != notification_status LIMIT {}".format(MAX)
-    update = "UPDATE notifications SET notification_status = status WHERE id in ({})".format(subq)
+    subq = "SELECT id FROM notifications WHERE cast (status as text) != notification_status LIMIT {}".format(MAX) # nosec B608 no user-controlled input
+    update = "UPDATE notifications SET notification_status = status WHERE id in ({})".format(subq) # nosec B608 no user-controlled input
    result = db.session.execute(subq).fetchall()

    while len(result) > 0:
@@ -211,9 +211,8 @@ def fix_notification_statuses_not_in_sync():
        db.session.commit()
        result = db.session.execute(subq).fetchall()

-    subq_hist = "SELECT id FROM notification_history WHERE cast (status as text) != notification_status LIMIT {}" \
-        .format(MAX)
-    update = "UPDATE notification_history SET notification_status = status WHERE id in ({})".format(subq_hist)
+    subq_hist = "SELECT id FROM notification_history WHERE cast (status as text) != notification_status LIMIT {}".format(MAX)  # nosec B608
+    update = "UPDATE notification_history SET notification_status = status WHERE id in ({})".format(subq_hist) # nosec B608 no user-controlled input
    result = db.session.execute(subq_hist).fetchall()

    while len(result) > 0:
@@ -545,7 +544,8 @@ def populate_organisation_agreement_details_from_file(file_name):

            current_app.logger.info(f"Updating {org.name}")

-            assert org.agreement_signed
+            if not org.agreement_signed:
+                raise RuntimeError('Agreement was not signed')

            org.agreement_signed_version = float(row[1])
            org.agreement_signed_on_behalf_of_name = row[2].strip()
--- a/app/config.py
+++ b/app/config.py
@@ -102,7 +102,7 @@ class Config(object):

    # DB conection string
    SQLALCHEMY_DATABASE_URI = os.environ.get('SQLALCHEMY_DATABASE_URI')
-    
+
    # AWS SMS
    AWS_PINPOINT_REGION = os.environ.get("AWS_PINPOINT_REGION")
    AWS_US_TOLL_FREE_NUMBER = os.environ.get("AWS_US_TOLL_FREE_NUMBER")
@@ -157,7 +157,7 @@ class Config(object):
    ONE_OFF_MESSAGE_FILENAME = 'Report'
    MAX_VERIFY_CODE_COUNT = 5
    MAX_FAILED_LOGIN_COUNT = 10
-    
+
    SES_STUB_URL = None # TODO: set to a URL in env and remove this to use a stubbed SES service

    # be careful increasing this size without being sure that we won't see slowness in pysftp
@@ -179,7 +179,7 @@ class Config(object):
    SMS_CODE_TEMPLATE_ID = '36fb0730-6259-4da1-8a80-c8de22ad4246'
    EMAIL_2FA_TEMPLATE_ID = '299726d2-dba6-42b8-8209-30e1d66ea164'
    NEW_USER_EMAIL_VERIFICATION_TEMPLATE_ID = 'ece42649-22a8-4d06-b87f-d52d5d3f0a27'
-    PASSWORD_RESET_TEMPLATE_ID = '474e9242-823b-4f99-813d-ed392e7f1201'
+    PASSWORD_RESET_TEMPLATE_ID = '474e9242-823b-4f99-813d-ed392e7f1201' # nosec B105 - this is not a password
    ALREADY_REGISTERED_EMAIL_TEMPLATE_ID = '0880fbb1-a0c6-46f0-9a8e-36c986381ceb'
    CHANGE_EMAIL_CONFIRMATION_TEMPLATE_ID = 'eb4d9930-87ab-4aef-9bce-786762687884'
    SERVICE_NOW_LIVE_TEMPLATE_ID = '618185c6-3636-49cd-b7d2-6f6f5eb3bdde'
@@ -426,7 +426,7 @@ class Development(Config):
    #     Config.GOVUK_ALERTS_CLIENT_ID: ['govuk-alerts-secret-key']
    # }

-    SECRET_KEY = 'dev-notify-secret-key'
+    SECRET_KEY = 'dev-notify-secret-key' # nosec B105 - this is only used in development
    DANGEROUS_SALT = 'dev-notify-salt'

    MMG_INBOUND_SMS_AUTH = ['testkey']
@@ -546,13 +546,13 @@ class Live(Config):
    INVALID_PDF_BUCKET_NAME = 'production-letters-invalid-pdf' # not created in gsa sandbox
    TRANSIENT_UPLOADED_LETTERS = 'production-transient-uploaded-letters' # not created in gsa sandbox
    LETTER_SANITISE_BUCKET_NAME = 'production-letters-sanitise' # not created in gsa sandbox
-    
+
    FROM_NUMBER = 'US Notify'
    API_RATE_LIMIT_ENABLED = True
    CHECK_PROXY_HEADER = True
    SES_STUB_URL = None
    CRONITOR_ENABLED = True
-    
+
    # DEBUG = True
    REDIS_ENABLED = os.environ.get('REDIS_ENABLED')

--- a/app/delivery/send_to_providers.py
+++ b/app/delivery/send_to_providers.py
@@ -188,7 +188,7 @@ def provider_to_use(notification_type, international=True):
        chosen_provider = active_providers[0]
    else:
        weights = [p.priority for p in active_providers]
-        chosen_provider = random.choices(active_providers, weights=weights)[0]
+        chosen_provider = random.choices(active_providers, weights=weights)[0] # nosec B311 - this is not security/cryptography related

    return notification_provider_clients.get_client_by_name_and_type(chosen_provider.identifier, notification_type)

--- a/app/models.py
+++ b/app/models.py
@@ -1647,7 +1647,7 @@ class Notification(db.Model):
        """
        # this should only ever be called for letter notifications - it makes no sense otherwise and I'd rather not
        # get the two code flows mixed up at all
-        assert self.notification_type == LETTER_TYPE
+        assert self.notification_type == LETTER_TYPE # nosec B101 - current calling code already validates the correct type

        if self.status in [NOTIFICATION_CREATED, NOTIFICATION_SENDING]:
            return NOTIFICATION_STATUS_LETTER_ACCEPTED
--- a/app/schema_validation/init.py
+++ b/app/schema_validation/init.py
@@ -108,7 +108,7 @@ def __format_message(e):
            error_path = e.path.popleft()
            # no need to catch IndexError exception explicity as
            # error_path is None if e.path has no items
-        except Exception:
+        except IndexError:
            pass
        return error_path

--- a/app/xml_schemas/init.py
+++ b/app/xml_schemas/init.py
@@ -1,6 +1,8 @@
 from pathlib import Path

-from lxml import etree
+from defusedxml.lxml import fromstring
+# there is no equivalent in defusedxml to validate a schema
+from lxml.etree import XMLSchema  # nosec B410


 def validate_xml(document, schema_file_name):
@@ -8,13 +10,6 @@ def validate_xml(document, schema_file_name):
    path = Path(__file__).resolve().parent / schema_file_name
    contents = path.read_text()

-    schema_root = etree.XML(contents.encode('utf-8'))
-    schema = etree.XMLSchema(schema_root)
-    parser = etree.XMLParser(schema=schema)
-
-    try:
-        etree.fromstring(document, parser)
-    except etree.XMLSyntaxError:
-        return False
-
-    return True
+    schema_root = fromstring(contents.encode('utf-8'))
+    schema = XMLSchema(schema_root)
+    return schema.validate(fromstring(document))
--- a/requirements.in
+++ b/requirements.in
@@ -6,7 +6,7 @@ celery[sqs]==5.2.6
 Flask-Bcrypt==1.0.1
 flask-marshmallow==0.14.0
 Flask-Migrate==3.1.0
-git+https://github.com/mitsuhiko/flask-sqlalchemy.git@500e732dd1b975a56ab06a46bd1a20a21e682262#egg=Flask-SQLAlchemy==2.3.2.dev20190108
+git+https://github.com/pallets-eco/flask-sqlalchemy.git@aa7a61a5357cf6f5dcc135d98c781192457aa6fa#egg=Flask-SQLAlchemy==2.5.1
 Flask==2.1.2
 click-datetime==0.2
 # Should be pinned until a new gunicorn release greater than 20.1.0 comes out. (Due to eventlet v0.33 compatibility issues)
@@ -14,15 +14,16 @@ git+https://github.com/benoitc/gunicorn.git@1299ea9e967a61ae2edebe191082fd169b86
 iso8601==1.0.2
 itsdangerous==2.1.2
 jsonschema[format]==4.5.1
-marshmallow-sqlalchemy==0.28.0
+marshmallow-sqlalchemy==0.28.1
 marshmallow==3.15.0
 psycopg2-binary==2.9.3
 PyJWT==2.4.0
-SQLAlchemy==1.4.36
+SQLAlchemy==1.4.40
 cachetools==5.1.0
 beautifulsoup4==4.11.1
 lxml==4.9.1
-Werkzeug==2.0.3  # pyup: <2.1.0 # later versions are not compatible with the version of flask-sqlalchemy we have pinned
+defusedxml==0.7.1
+Werkzeug==2.1.1
 python-dotenv==0.20.0

 notifications-python-client==6.3.0
@@ -30,7 +31,7 @@ notifications-python-client==6.3.0
 # PaaS
 awscli-cwlogs==1.4.6

-notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@56.0.0
+notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@56.0.2

 # gds-metrics requires prometheseus 0.2.0, override that requirement as 0.7.1 brings significant performance gains
 prometheus-client==0.14.1
--- a/requirements.txt
+++ b/requirements.txt
@@ -69,6 +69,8 @@ click-repl==0.2.0
    # via celery
 colorama==0.4.4
    # via awscli
+defusedxml==0.7.1
+    # via -r requirements.in
 deprecated==1.2.13
    # via redis
 dnspython==2.2.1
@@ -86,6 +88,7 @@ flask==2.1.2
    #   flask-marshmallow
    #   flask-migrate
    #   flask-redis
+    #   flask-sqlalchemy
    #   gds-metrics
    #   notifications-utils
 flask-bcrypt==1.0.1
@@ -96,7 +99,7 @@ flask-migrate==3.1.0
    # via -r requirements.in
 flask-redis==0.4.0
    # via notifications-utils
-flask-sqlalchemy @ git+https://github.com/mitsuhiko/flask-sqlalchemy.git@500e732dd1b975a56ab06a46bd1a20a21e682262
+flask-sqlalchemy @ git+https://github.com/pallets-eco/flask-sqlalchemy.git@aa7a61a5357cf6f5dcc135d98c781192457aa6fa
    # via
    #   -r requirements.in
    #   flask-migrate
@@ -118,6 +121,8 @@ idna==3.3
    # via
    #   jsonschema
    #   requests
+importlib-metadata==4.12.0
+    # via flask
 iso8601==1.0.2
    # via -r requirements.in
 isoduration==20.11.0
@@ -154,13 +159,13 @@ marshmallow==3.15.0
    #   -r requirements.in
    #   flask-marshmallow
    #   marshmallow-sqlalchemy
-marshmallow-sqlalchemy==0.28.0
+marshmallow-sqlalchemy==0.28.1
    # via -r requirements.in
 mistune==0.8.4
    # via notifications-utils
 notifications-python-client==6.3.0
    # via -r requirements.in
-notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@56.0.0
+notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@56.0.2
    # via -r requirements.in
 orderedset==2.0.3
    # via notifications-utils
@@ -168,6 +173,7 @@ packaging==21.3
    # via
    #   bleach
    #   marshmallow
+    #   marshmallow-sqlalchemy
    #   redis
 phonenumbers==8.12.48
    # via notifications-utils
@@ -245,13 +251,16 @@ smartypants==2.0.1
    # via notifications-utils
 soupsieve==2.3.2.post1
    # via beautifulsoup4
-sqlalchemy==1.4.36
+sqlalchemy==1.4.40
    # via
    #   -r requirements.in
    #   alembic
+    #   flask-sqlalchemy
    #   marshmallow-sqlalchemy
 statsd==3.3.0
    # via notifications-utils
+typing-extensions==4.3.0
+    # via pypdf2
 uri-template==1.2.0
    # via jsonschema
 urllib3==1.26.9
@@ -269,12 +278,14 @@ webcolors==1.12
    # via jsonschema
 webencodings==0.5.1
    # via bleach
-werkzeug==2.0.3
+werkzeug==2.1.1
    # via
    #   -r requirements.in
    #   flask
 wrapt==1.14.1
    # via deprecated
+zipp==3.8.1
+    # via importlib-metadata

 # The following packages are considered to be unsafe in a requirements file:
 # setuptools
--- a/requirements_for_test.txt
+++ b/requirements_for_test.txt
@@ -1,4 +1,4 @@
-r requirements.txt
+--requirement requirements.txt
 flake8==4.0.1
 flake8-bugbear==22.4.25
 isort==5.10.1
--- a/zap.conf
+++ b/zap.conf
@@ -0,0 +1,121 @@
+# zap-full-scan rule configuration file
+# Change WARN to IGNORE to ignore rule or FAIL to fail if rule matches
+# Active scan rules set to IGNORE will not be run which will speed up the scan
+# Only the rule identifiers are used - the names are just for info
+# You can add your own messages to each rule by appending them after a tab on each line.
+0	WARN	(Directory Browsing - Active/release)
+10003	WARN	(Vulnerable JS Library - Passive/release)
+10010	FAIL	(Cookie No HttpOnly Flag - Passive/release)
+10011	FAIL	(Cookie Without Secure Flag - Passive/release)
+10015	WARN	(Incomplete or No Cache-control Header Set - Passive/release)
+10016	FAIL	(Web Browser XSS Protection Not Enabled)
+10017	WARN	(Cross-Domain JavaScript Source File Inclusion - Passive/release)
+10019	WARN	(Content-Type Header Missing - Passive/release)
+10020	FAIL	(X-Frame-Options Header - Passive/release)
+10021	WARN	(X-Content-Type-Options Header Missing - Passive/release)
+10023	WARN	(Information Disclosure - Debug Error Messages - Passive/release)
+10024	FAIL	(Information Disclosure - Sensitive Information in URL - Passive/release)
+10025	FAIL	(Information Disclosure - Sensitive Information in HTTP Referrer Header - Passive/release)
+10026	WARN	(HTTP Parameter Override - Passive/beta)
+10027	WARN	(Information Disclosure - Suspicious Comments - Passive/release)
+10028	FAIL	(Open Redirect - Passive/beta)
+10029	WARN	(Cookie Poisoning - Passive/beta)
+10030	WARN	(User Controllable Charset - Passive/beta)
+10031	WARN	(User Controllable HTML Element Attribute (Potential XSS) - Passive/beta)
+10032	WARN	(Viewstate - Passive/release)
+10033	WARN	(Directory Browsing - Passive/beta)
+10034	WARN	(Heartbleed OpenSSL Vulnerability (Indicative) - Passive/beta)
+10035	FAIL	(Strict-Transport-Security Header - Passive/beta)
+10036	WARN	(HTTP Server Response Header - Passive/beta)
+10037	WARN	(Server Leaks Information via "X-Powered-By" HTTP Response Header Field(s) - Passive/release)
+10038	FAIL	(Content Security Policy (CSP) Header Not Set - Passive/beta)
+10039	WARN	(X-Backend-Server Header Information Leak - Passive/beta)
+10040	FAIL	(Secure Pages Include Mixed Content - Passive/release)
+10041	WARN	(HTTP to HTTPS Insecure Transition in Form Post - Passive/beta)
+10042	WARN	(HTTPS to HTTP Insecure Transition in Form Post - Passive/beta)
+10043	FAIL	(User Controllable JavaScript Event (XSS) - Passive/beta)
+10044	WARN	(Big Redirect Detected (Potential Sensitive Information Leak) - Passive/beta)
+10045	WARN	(Source Code Disclosure - /WEB-INF folder - Active/release)
+10047	WARN	(HTTPS Content Available via HTTP - Active/beta)
+10048	FAIL	(Remote Code Execution - Shell Shock - Active/beta)
+10050	WARN	(Retrieved from Cache - Passive/beta)
+10051	WARN	(Relative Path Confusion - Active/beta)
+10052	WARN	(X-ChromeLogger-Data (XCOLD) Header Information Leak - Passive/beta)
+10053	WARN	(Apache Range Header DoS (CVE-2011-3192) - Active/beta)
+10054	WARN	(Cookie without SameSite Attribute - Passive/release)
+10055	WARN	(CSP - Passive/release)
+10056	WARN	(X-Debug-Token Information Leak - Passive/release)
+10057	WARN	(Username Hash Found - Passive/release)
+10058	FAIL	(GET for POST - Active/beta)
+10061	WARN	(X-AspNet-Version Response Header - Passive/release)
+10062	FAIL	(PII Disclosure - Passive/beta)
+10095	IGNORE	(Backup File Disclosure - Active/beta)
+10096	WARN	(Timestamp Disclosure - Passive/release)
+10097	WARN	(Hash Disclosure - Passive/beta)
+10098	WARN	(Cross-Domain Misconfiguration - Passive/release)
+10104	WARN	(User Agent Fuzzer - Active/beta)
+10105	WARN	(Weak Authentication Method - Passive/release)
+10106	IGNORE	(HTTP Only Site - Active/beta)
+10107	WARN	(Httpoxy - Proxy Header Misuse - Active/beta)
+10108	WARN	(Reverse Tabnabbing - Passive/beta)
+10109	WARN	(Modern Web Application - Passive/beta)
+10202	FAIL	(Absence of Anti-CSRF Tokens - Passive/release)
+2	WARN	(Private IP Disclosure - Passive/release)
+20012	FAIL	(Anti-CSRF Tokens Check - Active/beta)
+20014	WARN	(HTTP Parameter Pollution - Active/beta)
+20015	WARN	(Heartbleed OpenSSL Vulnerability - Active/beta)
+20016	WARN	(Cross-Domain Misconfiguration - Active/beta)
+20017	FAIL	(Source Code Disclosure - CVE-2012-1823 - Active/beta)
+20018	FAIL	(Remote Code Execution - CVE-2012-1823 - Active/beta)
+20019	WARN	(External Redirect - Active/release)
+3	WARN	(Session ID in URL Rewrite - Passive/release)
+30001	WARN	(Buffer Overflow - Active/release)
+30002	WARN	(Format String Error - Active/release)
+30003	WARN	(Integer Overflow Error - Active/beta)
+40003	WARN	(CRLF Injection - Active/release)
+40008	WARN	(Parameter Tampering - Active/release)
+40009	WARN	(Server Side Include - Active/release)
+40012	FAIL	(Cross Site Scripting (Reflected) - Active/release)
+40013	FAIL	(Session Fixation - Active/beta)
+40014	FAIL	(Cross Site Scripting (Persistent) - Active/release)
+40016	FAIL	(Cross Site Scripting (Persistent) - Prime - Active/release)
+40017	FAIL	(Cross Site Scripting (Persistent) - Spider - Active/release)
+40018	FAIL	(SQL Injection - Active/release)
+40019	FAIL	(SQL Injection - MySQL - Active/beta)
+40020	FAIL	(SQL Injection - Hypersonic SQL - Active/beta)
+40021	FAIL	(SQL Injection - Oracle - Active/beta)
+40022	FAIL	(SQL Injection - PostgreSQL - Active/beta)
+40023	FAIL	(Possible Username Enumeration - Active/beta)
+40024	FAIL	(SQL Injection - SQLite - Active/beta)
+40025	FAIL	(Proxy Disclosure - Active/beta)
+40026	FAIL	(Cross Site Scripting (DOM Based) - Active/beta)
+40027	FAIL	(SQL Injection - MsSQL - Active/beta)
+40028	WARN	(ELMAH Information Leak - Active/release)
+40029	WARN	(Trace.axd Information Leak - Active/beta)
+40032	FAIL	(.htaccess Information Leak - Active/release)
+40034	FAIL	(.env Information Leak - Active/beta)
+40035	FAIL	(Hidden File Finder - Active/beta)
+41	FAIL	(Source Code Disclosure - Git  - Active/beta)
+42	WARN	(Source Code Disclosure - SVN - Active/beta)
+43	WARN	(Source Code Disclosure - File Inclusion - Active/beta)
+50000	WARN	(Script Active Scan Rules - Active/release)
+50001	WARN	(Script Passive Scan Rules - Passive/release)
+6	WARN	(Path Traversal - Active/release)
+7	WARN	(Remote File Inclusion - Active/release)
+90001	WARN	(Insecure JSF ViewState - Passive/release)
+90011	WARN	(Charset Mismatch - Passive/release)
+90017	WARN	(XSLT Injection - Active/beta)
+90019	WARN	(Server Side Code Injection - Active/release)
+90020	FAIL	(Remote OS Command Injection - Active/release)
+90021	WARN	(XPath Injection - Active/beta)
+90022	WARN	(Application Error Disclosure - Passive/release)
+90023	WARN	(XML External Entity Attack - Active/beta)
+90024	WARN	(Generic Padding Oracle - Active/beta)
+90025	WARN	(Expression Language Injection - Active/beta)
+90026	WARN	(SOAP Action Spoofing - Active/alpha)
+90027	IGNORE	(Cookie Slack Detector - Active/beta)
+90028	WARN	(Insecure HTTP Method - Active/beta)
+90029	WARN	(SOAP XML Injection - Active/alpha)
+90030	WARN	(WSDL File Detection - Passive/alpha)
+90033	WARN	(Loosely Scoped Cookie - Passive/release)
+90034	WARN	(Cloud Metadata Potentially Exposed - Active/beta)