diff --git a/.github/actions/setup-project/action.yml b/.github/actions/setup-project/action.yml new file mode 100644 index 000000000..003d8bf6e --- /dev/null +++ b/.github/actions/setup-project/action.yml @@ -0,0 +1,15 @@ +name: Set up project +description: Setup python & install dependencies +runs: + using: composite + steps: + - name: Install container dependencies + shell: bash + run: | + sudo apt-get update \ + && sudo apt-get install -y --no-install-recommends \ + libcurl4-openssl-dev + - name: Set up Python 3.9 + uses: actions/setup-python@v3 + with: + python-version: "3.9" diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 217e5b24d..b7340ec17 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -50,25 +50,75 @@ jobs: - 5432:5432 steps: - - name: Install container dependencies - run: | - sudo apt-get update \ - && sudo apt-get install -y --no-install-recommends \ - libcurl4-openssl-dev - uses: actions/checkout@v3 - - name: Set up Python 3.9 - uses: actions/setup-python@v3 - with: - python-version: "3.9" + - uses: ./.github/actions/setup-project - name: Install application dependencies run: make bootstrap env: SQLALCHEMY_DATABASE_TEST_URI: postgresql://user:password@localhost:5432/test_notification_api # - name: Run style checks # run: flake8 . - # - name: Check imports alphabetized - # run: isort --check-only ./app ./tests + - name: Check imports alphabetized + run: isort --check-only ./app ./tests - name: Run tests run: pytest -n4 --maxfail=10 env: SQLALCHEMY_DATABASE_TEST_URI: postgresql://user:password@localhost:5432/test_notification_api + + pip-audit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: ./.github/actions/setup-project + - uses: trailofbits/gh-action-pip-audit@v1.0.0 + with: + inputs: requirements.txt requirements_for_test.txt + ignore-vulns: PYSEC-2022-237 + + static-scan: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: ./.github/actions/setup-project + - name: Install bandit + run: pip install bandit + - name: Run scan + run: bandit -r app/ --confidence-level medium + + dynamic-scan: + runs-on: ubuntu-latest + services: + postgres: + image: postgres + env: + POSTGRES_USER: user + POSTGRES_PASSWORD: password + POSTGRES_DB: test_notification_api + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + # Maps tcp port 5432 on service container to the host + - 5432:5432 + steps: + - uses: actions/checkout@v3 + - uses: ./.github/actions/setup-project + - name: Install application dependencies + run: make bootstrap + env: + SQLALCHEMY_DATABASE_TEST_URI: postgresql://user:password@localhost:5432/test_notification_api + - name: Run server + run: make run-flask & + env: + SQLALCHEMY_DATABASE_TEST_URI: postgresql://user:password@localhost:5432/test_notification_api + - name: Run OWASP Baseline Scan + uses: zaproxy/action-api-scan@v0.1.1 + with: + docker_name: 'owasp/zap2docker-weekly' + target: 'http://localhost:6011/_status' + fail_action: true + allow_issue_writing: false + rules_file_name: 'zap.conf' + cmd_options: '-I' diff --git a/.github/workflows/daily_checks.yml b/.github/workflows/daily_checks.yml new file mode 100644 index 000000000..16ea19a80 --- /dev/null +++ b/.github/workflows/daily_checks.yml @@ -0,0 +1,93 @@ +name: Run daily scans + +on: + schedule: + # cron format: 'minute hour dayofmonth month dayofweek' + # this will run at noon UTC every day (7am EST / 8am EDT) + - cron: '0 12 * * *' + +permissions: + contents: read + +env: + DEBUG: True + ANTIVIRUS_ENABLED: 0 + NOTIFY_ENVIRONMENT: test + NOTIFICATION_QUEUE_PREFIX: local_dev_10x + STATSD_HOST: localhost + SES_STUB_URL: None + NOTIFY_APP_NAME: api + NOTIFY_EMAIL_DOMAIN: dispostable.com + NOTIFY_LOG_PATH: /workspace/logs/app.log + ADMIN_CLIENT_ID: notify-admin + ADMIN_CLIENT_SECRET: dev-notify-secret-key + GOVUK_ALERTS_CLIENT_ID: govuk-alerts + FLASK_APP: application.py + FLASK_ENV: development + WERKZEUG_DEBUG_PIN: off + ADMIN_BASE_URL: http://localhost:6012 + API_HOST_NAME: http://localhost:6011 + REDIS_URL: redis://localhost:6380 + REDIS_ENABLED: False + AWS_REGION: us-west-2 + AWS_PINPOINT_REGION: us-west-2 + AWS_US_TOLL_FREE_NUMBER: +18446120782 + +jobs: + pip-audit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: ./.github/actions/setup-project + - uses: trailofbits/gh-action-pip-audit@v1.0.0 + with: + inputs: requirements.txt requirements_for_test.txt + ignore-vulns: PYSEC-2022-237 + + static-scan: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: ./.github/actions/setup-project + - name: Install bandit + run: pip install bandit + - name: Run scan + run: bandit -r app/ --confidence-level medium + + dynamic-scan: + runs-on: ubuntu-latest + services: + postgres: + image: postgres + env: + POSTGRES_USER: user + POSTGRES_PASSWORD: password + POSTGRES_DB: test_notification_api + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + # Maps tcp port 5432 on service container to the host + - 5432:5432 + steps: + - uses: actions/checkout@v3 + - uses: ./.github/actions/setup-project + - name: Install application dependencies + run: make bootstrap + env: + SQLALCHEMY_DATABASE_TEST_URI: postgresql://user:password@localhost:5432/test_notification_api + - name: Run server + run: make run-flask & + env: + SQLALCHEMY_DATABASE_TEST_URI: postgresql://user:password@localhost:5432/test_notification_api + - name: Run OWASP Baseline Scan + uses: zaproxy/action-api-scan@v0.1.1 + with: + docker_name: 'owasp/zap2docker-weekly' + target: 'http://localhost:6011/_status' + fail_action: true + allow_issue_writing: false + rules_file_name: 'zap.conf' + cmd_options: '-I' diff --git a/Makefile b/Makefile index 7b13cced5..f3d47b438 100644 --- a/Makefile +++ b/Makefile @@ -77,6 +77,16 @@ freeze-requirements: ## Pin all requirements including sub dependencies into req pip install --upgrade pip-tools pip-compile requirements.in +.PHONY: audit +audit: + pip install --upgrade pip-audit + pip-audit -r requirements.txt -r requirements_for_test.txt -l --ignore-vuln PYSEC-2022-237 + +.PHONY: static-scan +static-scan: + pip install bandit + bandit -r app/ + .PHONY: clean clean: rm -rf node_modules cache target venv .coverage build tests/.cache ${CF_MANIFEST_PATH} diff --git a/README.md b/README.md index d93cfd0e9..564e6346f 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ Create the external docker network: `docker network create notify-network` Using the command palette (shift+cmd+p), search and select “Remote Containers: Open Folder in Container...” -When prompted, choose **devcontainer-api** folder (note: this is a *subfolder* of notification-api). This will startup the container in a new window (replacing the current one). +When prompted, choose **devcontainer-api** folder (note: this is a *subfolder* of notification-api). This will startup the container in a new window (replacing the current one). After this page loads, hit "show logs” in bottom-right. The first time this runs it will need to build the Docker image, which will likely take several minutes. @@ -39,7 +39,7 @@ Open another terminal and run the background tasks: ### `.env` file -Create and edit a .env file, based on sample.env. +Create and edit a .env file, based on sample.env. NOTE: when you change .env in the future, you'll need to rebuild the devcontainer for the change to take effect. Vscode _should_ detect the change and prompt you with a toast notification during a cached build. If not, you can find a manual rebuild in command pallette or just `docker rm` the notifications-api container. @@ -109,10 +109,19 @@ make bootstrap make test ``` +## To run a local OWASP scan + +1. Run `make run-flask` from within the dev container. +2. On your host machine run: + +``` +docker run -v $(pwd):/zap/wrk/:rw --network="notify-network" -t owasp/zap2docker-weekly zap-api-scan.py -t http://dev:6011/_status -f openapi -c zap.conf +``` + ## To run scheduled tasks ``` -# After scheduling some tasks, open a third terminal in your running devcontainer and run celery beat +# After scheduling some tasks, open a third terminal in your running devcontainer and run celery beat make run-celery-beat ``` diff --git a/app/__init__.py b/app/__init__.py index 9e975c8ce..2fffe2673 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -1,5 +1,5 @@ import os -import random +import secrets import string import time import uuid @@ -41,12 +41,13 @@ class SQLAlchemy(_SQLAlchemy): """We need to subclass SQLAlchemy in order to override create_engine options""" def apply_driver_hacks(self, app, info, options): - super().apply_driver_hacks(app, info, options) + sa_url, options = super().apply_driver_hacks(app, info, options) if 'connect_args' not in options: options['connect_args'] = {} options['connect_args']["options"] = "-c statement_timeout={}".format( int(app.config['SQLALCHEMY_STATEMENT_TIMEOUT']) * 1000 ) + return (sa_url, options) db = SQLAlchemy() @@ -352,7 +353,7 @@ def create_uuid(): def create_random_identifier(): - return ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(16)) + return ''.join(secrets.choice(string.ascii_uppercase + string.digits) for _ in range(16)) def setup_sqlalchemy_events(app): diff --git a/app/authentication/auth.py b/app/authentication/auth.py index 4c4b88596..301ed7853 100644 --- a/app/authentication/auth.py +++ b/app/authentication/auth.py @@ -18,7 +18,7 @@ from sqlalchemy.orm.exc import NoResultFound from app.serialised_models import SerialisedService -GENERAL_TOKEN_ERROR_MESSAGE = 'Invalid token: make sure your API token matches the example at https://docs.notifications.service.gov.uk/rest-api.html#authorisation-header' # noqa +GENERAL_TOKEN_ERROR_MESSAGE = 'Invalid token: make sure your API token matches the example at https://docs.notifications.service.gov.uk/rest-api.html#authorisation-header' # nosec B105 AUTH_DB_CONNECTION_DURATION_SECONDS = Histogram( 'auth_db_connection_duration_seconds', diff --git a/app/celery/process_sms_client_response_tasks.py b/app/celery/process_sms_client_response_tasks.py index 3676fe72a..e7726f305 100644 --- a/app/celery/process_sms_client_response_tasks.py +++ b/app/celery/process_sms_client_response_tasks.py @@ -14,13 +14,13 @@ from app.notifications.notifications_ses_callback import ( check_and_queue_callback_task, ) -# sms_response_mapper = { -# 'MMG': get_mmg_responses, -# 'Firetext': get_firetext_responses, -# } +sms_response_mapper = { + # 'MMG': get_mmg_responses, + # 'Firetext': get_firetext_responses, +} -gUpdate with new providers") +# gUpdate with new providers") @notify_celery.task(bind=True, name="process-sms-client-response", max_retries=5, default_retry_delay=300) def process_sms_client_response(self, status, provider_reference, client_name, detailed_status_code=None): # validate reference diff --git a/app/celery/research_mode_tasks.py b/app/celery/research_mode_tasks.py index e937739f5..ac99bc842 100644 --- a/app/celery/research_mode_tasks.py +++ b/app/celery/research_mode_tasks.py @@ -124,7 +124,7 @@ def create_fake_letter_response_file(self, reference): dvla_response_data = '{}|Sent|0|Sorted'.format(reference) # try and find a filename that hasn't been taken yet - from a random time within the last 30 seconds - for i in sorted(range(30), key=lambda _: random.random()): + for i in sorted(range(30), key=lambda _: random.random()): # nosec B311 - not security related upload_file_name = 'NOTIFY-{}-RSP.TXT'.format((now - timedelta(seconds=i)).strftime('%Y%m%d%H%M%S')) if not file_exists(current_app.config['DVLA_RESPONSE_BUCKET_NAME'], upload_file_name): break diff --git a/app/clients/__init__.py b/app/clients/__init__.py index a4a6d2298..71c66bed3 100644 --- a/app/clients/__init__.py +++ b/app/clients/__init__.py @@ -38,8 +38,8 @@ class NotificationProviderClients(object): return self.email_clients.get(name) def get_client_by_name_and_type(self, name, notification_type): - assert notification_type in ['email', 'sms'] - + assert notification_type in ['email', 'sms'] # nosec B101 + if notification_type == 'email': return self.get_email_client(name) diff --git a/app/commands.py b/app/commands.py index e217d9e29..ad944178d 100644 --- a/app/commands.py +++ b/app/commands.py @@ -151,8 +151,8 @@ def backfill_notification_statuses(): `Notification._status_enum` """ LIMIT = 250000 - subq = "SELECT id FROM notification_history WHERE notification_status is NULL LIMIT {}".format(LIMIT) - update = "UPDATE notification_history SET notification_status = status WHERE id in ({})".format(subq) + subq = "SELECT id FROM notification_history WHERE notification_status is NULL LIMIT {}".format(LIMIT) # nosec B608 no user-controlled input + update = "UPDATE notification_history SET notification_status = status WHERE id in ({})".format(subq) # nosec B608 no user-controlled input result = db.session.execute(subq).fetchall() while len(result) > 0: @@ -169,7 +169,7 @@ def update_notification_international_flag(): """ # 250,000 rows takes 30 seconds to update. subq = "select id from notifications where international is null limit 250000" - update = "update notifications set international = False where id in ({})".format(subq) + update = "update notifications set international = False where id in ({})".format(subq) # nosec B608 no user-controlled input result = db.session.execute(subq).fetchall() while len(result) > 0: @@ -180,7 +180,7 @@ def update_notification_international_flag(): # Now update notification_history subq_history = "select id from notification_history where international is null limit 250000" - update_history = "update notification_history set international = False where id in ({})".format(subq_history) + update_history = "update notification_history set international = False where id in ({})".format(subq_history) # nosec B608 no user-controlled input result_history = db.session.execute(subq_history).fetchall() while len(result_history) > 0: db.session.execute(update_history) @@ -201,8 +201,8 @@ def fix_notification_statuses_not_in_sync(): """ MAX = 10000 - subq = "SELECT id FROM notifications WHERE cast (status as text) != notification_status LIMIT {}".format(MAX) - update = "UPDATE notifications SET notification_status = status WHERE id in ({})".format(subq) + subq = "SELECT id FROM notifications WHERE cast (status as text) != notification_status LIMIT {}".format(MAX) # nosec B608 no user-controlled input + update = "UPDATE notifications SET notification_status = status WHERE id in ({})".format(subq) # nosec B608 no user-controlled input result = db.session.execute(subq).fetchall() while len(result) > 0: @@ -211,9 +211,8 @@ def fix_notification_statuses_not_in_sync(): db.session.commit() result = db.session.execute(subq).fetchall() - subq_hist = "SELECT id FROM notification_history WHERE cast (status as text) != notification_status LIMIT {}" \ - .format(MAX) - update = "UPDATE notification_history SET notification_status = status WHERE id in ({})".format(subq_hist) + subq_hist = "SELECT id FROM notification_history WHERE cast (status as text) != notification_status LIMIT {}".format(MAX) # nosec B608 + update = "UPDATE notification_history SET notification_status = status WHERE id in ({})".format(subq_hist) # nosec B608 no user-controlled input result = db.session.execute(subq_hist).fetchall() while len(result) > 0: @@ -545,7 +544,8 @@ def populate_organisation_agreement_details_from_file(file_name): current_app.logger.info(f"Updating {org.name}") - assert org.agreement_signed + if not org.agreement_signed: + raise RuntimeError('Agreement was not signed') org.agreement_signed_version = float(row[1]) org.agreement_signed_on_behalf_of_name = row[2].strip() diff --git a/app/config.py b/app/config.py index d06431ad4..d1ab7e683 100644 --- a/app/config.py +++ b/app/config.py @@ -102,7 +102,7 @@ class Config(object): # DB conection string SQLALCHEMY_DATABASE_URI = os.environ.get('SQLALCHEMY_DATABASE_URI') - + # AWS SMS AWS_PINPOINT_REGION = os.environ.get("AWS_PINPOINT_REGION") AWS_US_TOLL_FREE_NUMBER = os.environ.get("AWS_US_TOLL_FREE_NUMBER") @@ -157,7 +157,7 @@ class Config(object): ONE_OFF_MESSAGE_FILENAME = 'Report' MAX_VERIFY_CODE_COUNT = 5 MAX_FAILED_LOGIN_COUNT = 10 - + SES_STUB_URL = None # TODO: set to a URL in env and remove this to use a stubbed SES service # be careful increasing this size without being sure that we won't see slowness in pysftp @@ -179,7 +179,7 @@ class Config(object): SMS_CODE_TEMPLATE_ID = '36fb0730-6259-4da1-8a80-c8de22ad4246' EMAIL_2FA_TEMPLATE_ID = '299726d2-dba6-42b8-8209-30e1d66ea164' NEW_USER_EMAIL_VERIFICATION_TEMPLATE_ID = 'ece42649-22a8-4d06-b87f-d52d5d3f0a27' - PASSWORD_RESET_TEMPLATE_ID = '474e9242-823b-4f99-813d-ed392e7f1201' + PASSWORD_RESET_TEMPLATE_ID = '474e9242-823b-4f99-813d-ed392e7f1201' # nosec B105 - this is not a password ALREADY_REGISTERED_EMAIL_TEMPLATE_ID = '0880fbb1-a0c6-46f0-9a8e-36c986381ceb' CHANGE_EMAIL_CONFIRMATION_TEMPLATE_ID = 'eb4d9930-87ab-4aef-9bce-786762687884' SERVICE_NOW_LIVE_TEMPLATE_ID = '618185c6-3636-49cd-b7d2-6f6f5eb3bdde' @@ -426,7 +426,7 @@ class Development(Config): # Config.GOVUK_ALERTS_CLIENT_ID: ['govuk-alerts-secret-key'] # } - SECRET_KEY = 'dev-notify-secret-key' + SECRET_KEY = 'dev-notify-secret-key' # nosec B105 - this is only used in development DANGEROUS_SALT = 'dev-notify-salt' MMG_INBOUND_SMS_AUTH = ['testkey'] @@ -546,13 +546,13 @@ class Live(Config): INVALID_PDF_BUCKET_NAME = 'production-letters-invalid-pdf' # not created in gsa sandbox TRANSIENT_UPLOADED_LETTERS = 'production-transient-uploaded-letters' # not created in gsa sandbox LETTER_SANITISE_BUCKET_NAME = 'production-letters-sanitise' # not created in gsa sandbox - + FROM_NUMBER = 'US Notify' API_RATE_LIMIT_ENABLED = True CHECK_PROXY_HEADER = True SES_STUB_URL = None CRONITOR_ENABLED = True - + # DEBUG = True REDIS_ENABLED = os.environ.get('REDIS_ENABLED') diff --git a/app/delivery/send_to_providers.py b/app/delivery/send_to_providers.py index 84e3f0f04..d546cbc0c 100644 --- a/app/delivery/send_to_providers.py +++ b/app/delivery/send_to_providers.py @@ -188,7 +188,7 @@ def provider_to_use(notification_type, international=True): chosen_provider = active_providers[0] else: weights = [p.priority for p in active_providers] - chosen_provider = random.choices(active_providers, weights=weights)[0] + chosen_provider = random.choices(active_providers, weights=weights)[0] # nosec B311 - this is not security/cryptography related return notification_provider_clients.get_client_by_name_and_type(chosen_provider.identifier, notification_type) diff --git a/app/models.py b/app/models.py index 41abfc6d9..ff6a8a8d9 100644 --- a/app/models.py +++ b/app/models.py @@ -1647,7 +1647,7 @@ class Notification(db.Model): """ # this should only ever be called for letter notifications - it makes no sense otherwise and I'd rather not # get the two code flows mixed up at all - assert self.notification_type == LETTER_TYPE + assert self.notification_type == LETTER_TYPE # nosec B101 - current calling code already validates the correct type if self.status in [NOTIFICATION_CREATED, NOTIFICATION_SENDING]: return NOTIFICATION_STATUS_LETTER_ACCEPTED diff --git a/app/schema_validation/__init__.py b/app/schema_validation/__init__.py index 13dd5de87..7e813f4d0 100644 --- a/app/schema_validation/__init__.py +++ b/app/schema_validation/__init__.py @@ -108,7 +108,7 @@ def __format_message(e): error_path = e.path.popleft() # no need to catch IndexError exception explicity as # error_path is None if e.path has no items - except Exception: + except IndexError: pass return error_path diff --git a/app/xml_schemas/__init__.py b/app/xml_schemas/__init__.py index 8f02f994e..fc02965d0 100644 --- a/app/xml_schemas/__init__.py +++ b/app/xml_schemas/__init__.py @@ -1,6 +1,8 @@ from pathlib import Path -from lxml import etree +from defusedxml.lxml import fromstring +# there is no equivalent in defusedxml to validate a schema +from lxml.etree import XMLSchema # nosec B410 def validate_xml(document, schema_file_name): @@ -8,13 +10,6 @@ def validate_xml(document, schema_file_name): path = Path(__file__).resolve().parent / schema_file_name contents = path.read_text() - schema_root = etree.XML(contents.encode('utf-8')) - schema = etree.XMLSchema(schema_root) - parser = etree.XMLParser(schema=schema) - - try: - etree.fromstring(document, parser) - except etree.XMLSyntaxError: - return False - - return True + schema_root = fromstring(contents.encode('utf-8')) + schema = XMLSchema(schema_root) + return schema.validate(fromstring(document)) diff --git a/requirements.in b/requirements.in index db5cdb9b3..6f6ff71c5 100644 --- a/requirements.in +++ b/requirements.in @@ -6,7 +6,7 @@ celery[sqs]==5.2.6 Flask-Bcrypt==1.0.1 flask-marshmallow==0.14.0 Flask-Migrate==3.1.0 -git+https://github.com/mitsuhiko/flask-sqlalchemy.git@500e732dd1b975a56ab06a46bd1a20a21e682262#egg=Flask-SQLAlchemy==2.3.2.dev20190108 +git+https://github.com/pallets-eco/flask-sqlalchemy.git@aa7a61a5357cf6f5dcc135d98c781192457aa6fa#egg=Flask-SQLAlchemy==2.5.1 Flask==2.1.2 click-datetime==0.2 # Should be pinned until a new gunicorn release greater than 20.1.0 comes out. (Due to eventlet v0.33 compatibility issues) @@ -14,15 +14,16 @@ git+https://github.com/benoitc/gunicorn.git@1299ea9e967a61ae2edebe191082fd169b86 iso8601==1.0.2 itsdangerous==2.1.2 jsonschema[format]==4.5.1 -marshmallow-sqlalchemy==0.28.0 +marshmallow-sqlalchemy==0.28.1 marshmallow==3.15.0 psycopg2-binary==2.9.3 PyJWT==2.4.0 -SQLAlchemy==1.4.36 +SQLAlchemy==1.4.40 cachetools==5.1.0 beautifulsoup4==4.11.1 lxml==4.9.1 -Werkzeug==2.0.3 # pyup: <2.1.0 # later versions are not compatible with the version of flask-sqlalchemy we have pinned +defusedxml==0.7.1 +Werkzeug==2.1.1 python-dotenv==0.20.0 notifications-python-client==6.3.0 @@ -30,7 +31,7 @@ notifications-python-client==6.3.0 # PaaS awscli-cwlogs==1.4.6 -notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@56.0.0 +notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@56.0.2 # gds-metrics requires prometheseus 0.2.0, override that requirement as 0.7.1 brings significant performance gains prometheus-client==0.14.1 diff --git a/requirements.txt b/requirements.txt index 867feacea..8003b54e1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -69,6 +69,8 @@ click-repl==0.2.0 # via celery colorama==0.4.4 # via awscli +defusedxml==0.7.1 + # via -r requirements.in deprecated==1.2.13 # via redis dnspython==2.2.1 @@ -86,6 +88,7 @@ flask==2.1.2 # flask-marshmallow # flask-migrate # flask-redis + # flask-sqlalchemy # gds-metrics # notifications-utils flask-bcrypt==1.0.1 @@ -96,7 +99,7 @@ flask-migrate==3.1.0 # via -r requirements.in flask-redis==0.4.0 # via notifications-utils -flask-sqlalchemy @ git+https://github.com/mitsuhiko/flask-sqlalchemy.git@500e732dd1b975a56ab06a46bd1a20a21e682262 +flask-sqlalchemy @ git+https://github.com/pallets-eco/flask-sqlalchemy.git@aa7a61a5357cf6f5dcc135d98c781192457aa6fa # via # -r requirements.in # flask-migrate @@ -118,6 +121,8 @@ idna==3.3 # via # jsonschema # requests +importlib-metadata==4.12.0 + # via flask iso8601==1.0.2 # via -r requirements.in isoduration==20.11.0 @@ -154,13 +159,13 @@ marshmallow==3.15.0 # -r requirements.in # flask-marshmallow # marshmallow-sqlalchemy -marshmallow-sqlalchemy==0.28.0 +marshmallow-sqlalchemy==0.28.1 # via -r requirements.in mistune==0.8.4 # via notifications-utils notifications-python-client==6.3.0 # via -r requirements.in -notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@56.0.0 +notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@56.0.2 # via -r requirements.in orderedset==2.0.3 # via notifications-utils @@ -168,6 +173,7 @@ packaging==21.3 # via # bleach # marshmallow + # marshmallow-sqlalchemy # redis phonenumbers==8.12.48 # via notifications-utils @@ -245,13 +251,16 @@ smartypants==2.0.1 # via notifications-utils soupsieve==2.3.2.post1 # via beautifulsoup4 -sqlalchemy==1.4.36 +sqlalchemy==1.4.40 # via # -r requirements.in # alembic + # flask-sqlalchemy # marshmallow-sqlalchemy statsd==3.3.0 # via notifications-utils +typing-extensions==4.3.0 + # via pypdf2 uri-template==1.2.0 # via jsonschema urllib3==1.26.9 @@ -269,12 +278,14 @@ webcolors==1.12 # via jsonschema webencodings==0.5.1 # via bleach -werkzeug==2.0.3 +werkzeug==2.1.1 # via # -r requirements.in # flask wrapt==1.14.1 # via deprecated +zipp==3.8.1 + # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/requirements_for_test.txt b/requirements_for_test.txt index d7450055e..3c6756bc1 100644 --- a/requirements_for_test.txt +++ b/requirements_for_test.txt @@ -1,4 +1,4 @@ --r requirements.txt +--requirement requirements.txt flake8==4.0.1 flake8-bugbear==22.4.25 isort==5.10.1 diff --git a/zap.conf b/zap.conf new file mode 100644 index 000000000..f4e88ff07 --- /dev/null +++ b/zap.conf @@ -0,0 +1,121 @@ +# zap-full-scan rule configuration file +# Change WARN to IGNORE to ignore rule or FAIL to fail if rule matches +# Active scan rules set to IGNORE will not be run which will speed up the scan +# Only the rule identifiers are used - the names are just for info +# You can add your own messages to each rule by appending them after a tab on each line. +0 WARN (Directory Browsing - Active/release) +10003 WARN (Vulnerable JS Library - Passive/release) +10010 FAIL (Cookie No HttpOnly Flag - Passive/release) +10011 FAIL (Cookie Without Secure Flag - Passive/release) +10015 WARN (Incomplete or No Cache-control Header Set - Passive/release) +10016 FAIL (Web Browser XSS Protection Not Enabled) +10017 WARN (Cross-Domain JavaScript Source File Inclusion - Passive/release) +10019 WARN (Content-Type Header Missing - Passive/release) +10020 FAIL (X-Frame-Options Header - Passive/release) +10021 WARN (X-Content-Type-Options Header Missing - Passive/release) +10023 WARN (Information Disclosure - Debug Error Messages - Passive/release) +10024 FAIL (Information Disclosure - Sensitive Information in URL - Passive/release) +10025 FAIL (Information Disclosure - Sensitive Information in HTTP Referrer Header - Passive/release) +10026 WARN (HTTP Parameter Override - Passive/beta) +10027 WARN (Information Disclosure - Suspicious Comments - Passive/release) +10028 FAIL (Open Redirect - Passive/beta) +10029 WARN (Cookie Poisoning - Passive/beta) +10030 WARN (User Controllable Charset - Passive/beta) +10031 WARN (User Controllable HTML Element Attribute (Potential XSS) - Passive/beta) +10032 WARN (Viewstate - Passive/release) +10033 WARN (Directory Browsing - Passive/beta) +10034 WARN (Heartbleed OpenSSL Vulnerability (Indicative) - Passive/beta) +10035 FAIL (Strict-Transport-Security Header - Passive/beta) +10036 WARN (HTTP Server Response Header - Passive/beta) +10037 WARN (Server Leaks Information via "X-Powered-By" HTTP Response Header Field(s) - Passive/release) +10038 FAIL (Content Security Policy (CSP) Header Not Set - Passive/beta) +10039 WARN (X-Backend-Server Header Information Leak - Passive/beta) +10040 FAIL (Secure Pages Include Mixed Content - Passive/release) +10041 WARN (HTTP to HTTPS Insecure Transition in Form Post - Passive/beta) +10042 WARN (HTTPS to HTTP Insecure Transition in Form Post - Passive/beta) +10043 FAIL (User Controllable JavaScript Event (XSS) - Passive/beta) +10044 WARN (Big Redirect Detected (Potential Sensitive Information Leak) - Passive/beta) +10045 WARN (Source Code Disclosure - /WEB-INF folder - Active/release) +10047 WARN (HTTPS Content Available via HTTP - Active/beta) +10048 FAIL (Remote Code Execution - Shell Shock - Active/beta) +10050 WARN (Retrieved from Cache - Passive/beta) +10051 WARN (Relative Path Confusion - Active/beta) +10052 WARN (X-ChromeLogger-Data (XCOLD) Header Information Leak - Passive/beta) +10053 WARN (Apache Range Header DoS (CVE-2011-3192) - Active/beta) +10054 WARN (Cookie without SameSite Attribute - Passive/release) +10055 WARN (CSP - Passive/release) +10056 WARN (X-Debug-Token Information Leak - Passive/release) +10057 WARN (Username Hash Found - Passive/release) +10058 FAIL (GET for POST - Active/beta) +10061 WARN (X-AspNet-Version Response Header - Passive/release) +10062 FAIL (PII Disclosure - Passive/beta) +10095 IGNORE (Backup File Disclosure - Active/beta) +10096 WARN (Timestamp Disclosure - Passive/release) +10097 WARN (Hash Disclosure - Passive/beta) +10098 WARN (Cross-Domain Misconfiguration - Passive/release) +10104 WARN (User Agent Fuzzer - Active/beta) +10105 WARN (Weak Authentication Method - Passive/release) +10106 IGNORE (HTTP Only Site - Active/beta) +10107 WARN (Httpoxy - Proxy Header Misuse - Active/beta) +10108 WARN (Reverse Tabnabbing - Passive/beta) +10109 WARN (Modern Web Application - Passive/beta) +10202 FAIL (Absence of Anti-CSRF Tokens - Passive/release) +2 WARN (Private IP Disclosure - Passive/release) +20012 FAIL (Anti-CSRF Tokens Check - Active/beta) +20014 WARN (HTTP Parameter Pollution - Active/beta) +20015 WARN (Heartbleed OpenSSL Vulnerability - Active/beta) +20016 WARN (Cross-Domain Misconfiguration - Active/beta) +20017 FAIL (Source Code Disclosure - CVE-2012-1823 - Active/beta) +20018 FAIL (Remote Code Execution - CVE-2012-1823 - Active/beta) +20019 WARN (External Redirect - Active/release) +3 WARN (Session ID in URL Rewrite - Passive/release) +30001 WARN (Buffer Overflow - Active/release) +30002 WARN (Format String Error - Active/release) +30003 WARN (Integer Overflow Error - Active/beta) +40003 WARN (CRLF Injection - Active/release) +40008 WARN (Parameter Tampering - Active/release) +40009 WARN (Server Side Include - Active/release) +40012 FAIL (Cross Site Scripting (Reflected) - Active/release) +40013 FAIL (Session Fixation - Active/beta) +40014 FAIL (Cross Site Scripting (Persistent) - Active/release) +40016 FAIL (Cross Site Scripting (Persistent) - Prime - Active/release) +40017 FAIL (Cross Site Scripting (Persistent) - Spider - Active/release) +40018 FAIL (SQL Injection - Active/release) +40019 FAIL (SQL Injection - MySQL - Active/beta) +40020 FAIL (SQL Injection - Hypersonic SQL - Active/beta) +40021 FAIL (SQL Injection - Oracle - Active/beta) +40022 FAIL (SQL Injection - PostgreSQL - Active/beta) +40023 FAIL (Possible Username Enumeration - Active/beta) +40024 FAIL (SQL Injection - SQLite - Active/beta) +40025 FAIL (Proxy Disclosure - Active/beta) +40026 FAIL (Cross Site Scripting (DOM Based) - Active/beta) +40027 FAIL (SQL Injection - MsSQL - Active/beta) +40028 WARN (ELMAH Information Leak - Active/release) +40029 WARN (Trace.axd Information Leak - Active/beta) +40032 FAIL (.htaccess Information Leak - Active/release) +40034 FAIL (.env Information Leak - Active/beta) +40035 FAIL (Hidden File Finder - Active/beta) +41 FAIL (Source Code Disclosure - Git - Active/beta) +42 WARN (Source Code Disclosure - SVN - Active/beta) +43 WARN (Source Code Disclosure - File Inclusion - Active/beta) +50000 WARN (Script Active Scan Rules - Active/release) +50001 WARN (Script Passive Scan Rules - Passive/release) +6 WARN (Path Traversal - Active/release) +7 WARN (Remote File Inclusion - Active/release) +90001 WARN (Insecure JSF ViewState - Passive/release) +90011 WARN (Charset Mismatch - Passive/release) +90017 WARN (XSLT Injection - Active/beta) +90019 WARN (Server Side Code Injection - Active/release) +90020 FAIL (Remote OS Command Injection - Active/release) +90021 WARN (XPath Injection - Active/beta) +90022 WARN (Application Error Disclosure - Passive/release) +90023 WARN (XML External Entity Attack - Active/beta) +90024 WARN (Generic Padding Oracle - Active/beta) +90025 WARN (Expression Language Injection - Active/beta) +90026 WARN (SOAP Action Spoofing - Active/alpha) +90027 IGNORE (Cookie Slack Detector - Active/beta) +90028 WARN (Insecure HTTP Method - Active/beta) +90029 WARN (SOAP XML Injection - Active/alpha) +90030 WARN (WSDL File Detection - Passive/alpha) +90033 WARN (Loosely Scoped Cookie - Passive/release) +90034 WARN (Cloud Metadata Potentially Exposed - Active/beta)