From d27401c7a084e35be474d831559cddc929e8d15d Mon Sep 17 00:00:00 2001 From: stvnrlly Date: Wed, 26 Oct 2022 14:05:37 +0000 Subject: [PATCH] more pipenv transition --- .github/workflows/checks.yml | 2 + .github/workflows/daily_checks.yml | 2 + Makefile | 50 ++--- Pipfile.lock | 55 ++++++ README.md | 13 +- requirements.in | 39 ---- requirements.txt | 297 ----------------------------- requirements_for_test.txt | 14 -- 8 files changed, 80 insertions(+), 392 deletions(-) delete mode 100644 requirements.in delete mode 100644 requirements.txt delete mode 100644 requirements_for_test.txt diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 57e11688e..6fcdf8958 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -71,6 +71,8 @@ jobs: steps: - uses: actions/checkout@v3 - uses: ./.github/actions/setup-project + - name: Create requirements.txt + run: pipenv requirements - uses: trailofbits/gh-action-pip-audit@v1.0.0 with: inputs: requirements.txt diff --git a/.github/workflows/daily_checks.yml b/.github/workflows/daily_checks.yml index 06dd0bc19..1449ef3d5 100644 --- a/.github/workflows/daily_checks.yml +++ b/.github/workflows/daily_checks.yml @@ -38,6 +38,8 @@ jobs: steps: - uses: actions/checkout@v3 - uses: ./.github/actions/setup-project + - name: Create requirements.txt + run: pipenv requirements - uses: trailofbits/gh-action-pip-audit@v1.0.0 with: inputs: requirements.txt diff --git a/Makefile b/Makefile index fcd1ce054..7bc3da75c 100644 --- a/Makefile +++ b/Makefile @@ -7,18 +7,14 @@ APP_VERSION_FILE = app/version.py GIT_BRANCH ?= $(shell git symbolic-ref --short HEAD 2> /dev/null || echo "detached") GIT_COMMIT ?= $(shell git rev-parse HEAD) -CF_SPACE ?= ${DEPLOY_ENV} -CF_HOME ?= ${HOME} -$(eval export CF_HOME) - - ## DEVELOPMENT .PHONY: bootstrap -bootstrap: generate-version-file ## Set up everything to run the app - pip3 install -r requirements_for_test.txt +bootstrap: ## Set up everything to run the app + generate-version-file + pipenv install ---dev createdb notification_api || true - (flask db upgrade) || true + (pipenv run flask db upgrade) || true .PHONY: bootstrap-with-docker bootstrap-with-docker: ## Build the image to run the app in Docker @@ -26,31 +22,23 @@ bootstrap-with-docker: ## Build the image to run the app in Docker .PHONY: run-flask run-flask: ## Run flask - flask run -p 6011 --host=0.0.0.0 + pipenv run flask run -p 6011 --host=0.0.0.0 .PHONY: run-celery run-celery: ## Run celery, TODO remove purge for staging/prod - celery -A run_celery.notify_celery purge -f - celery \ + pipenv run celery -A run_celery.notify_celery purge -f + pipenv run celery \ -A run_celery.notify_celery worker \ --pidfile="/tmp/celery.pid" \ --loglevel=INFO \ --concurrency=4 -.PHONY: run-celery-with-docker -run-celery-with-docker: ## Run celery in Docker container (useful if you can't install pycurl locally) - ./scripts/run_with_docker.sh make run-celery - .PHONY: run-celery-beat run-celery-beat: ## Run celery beat - celery \ + pipenv run celery \ -A run_celery.notify_celery beat \ --loglevel=INFO -.PHONY: run-celery-beat-with-docker -run-celery-beat-with-docker: ## Run celery beat in Docker container (useful if you can't install pycurl locally) - ./scripts/run_with_docker.sh make run-celery-beat - .PHONY: help help: @cat $(MAKEFILE_LIST) | grep -E '^[a-zA-Z_-]+:.*?## .*$$' | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' @@ -67,12 +55,14 @@ test: ## Run tests .PHONY: freeze-requirements freeze-requirements: ## Pin all requirements including sub dependencies into requirements.txt - pip install --upgrade pip-tools - pip-compile requirements.in + pipenv lock + pipenv requirements .PHONY: audit audit: pip install --upgrade pip-audit + pipenv requirements > requirements.txt + pipenv requirements --dev > requirements_for_test.txt pip-audit -r requirements.txt -l --ignore-vuln PYSEC-2022-237 -pip-audit -r requirements_for_test.txt -l @@ -88,22 +78,6 @@ clean: ## DEPLOYMENT -.PHONY: cf-login -cf-login: ## Log in to Cloud Foundry - $(if ${CF_USERNAME},,$(error Must specify CF_USERNAME)) - $(if ${CF_PASSWORD},,$(error Must specify CF_PASSWORD)) - $(if ${CF_SPACE},,$(error Must specify CF_SPACE)) - @echo "Logging in to Cloud Foundry on ${CF_API}" - @cf login -a "${CF_API}" -u ${CF_USERNAME} -p "${CF_PASSWORD}" -o "${CF_ORG}" -s "${CF_SPACE}" - -.PHONY: cf-check-api-db-migration-task -cf-check-api-db-migration-task: ## Get the status for the last notifications-api task - @cf curl /v3/apps/`cf app --guid notifications-api`/tasks?order_by=-created_at | jq -r ".resources[0].state" - -.PHONY: check-if-migrations-to-run -check-if-migrations-to-run: - @echo $(shell python3 scripts/check_if_new_migration.py) - # .PHONY: cf-deploy-failwhale # cf-deploy-failwhale: # $(if ${CF_SPACE},,$(error Must target space, eg `make preview cf-deploy-failwhale`)) diff --git a/Pipfile.lock b/Pipfile.lock index 8cd0bf67e..565227cdb 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,11 @@ { "_meta": { "hash": { +<<<<<<< Updated upstream "sha256": "eca9a39871c8db3e82fb384c39afc89fdc3c145c87653f5090927e578618ce11" +======= + "sha256": "ce99b649bae4b10b084aacc936b72e01957eb9354cbc420d684590133c2da004" +>>>>>>> Stashed changes }, "pipfile-spec": 6, "requires": { @@ -370,11 +374,19 @@ "version": "==0.4.0" }, "flask-sqlalchemy": { +<<<<<<< Updated upstream +======= + "git": "https://github.com/pallets-eco/flask-sqlalchemy.git", +>>>>>>> Stashed changes "hashes": [ "sha256:2bda44b43e7cacb15d4e05ff3cc1f8bc97936cc464623424102bfc2c35e95912", "sha256:f12c3d4cc5cc7fdcc148b9527ea05671718c3ea45d50c7e732cceb33f574b390" ], +<<<<<<< Updated upstream "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", +======= + "ref": "aa7a61a5357cf6f5dcc135d98c781192457aa6fa", +>>>>>>> Stashed changes "version": "==2.5.1" }, "flask-sqlalchemy==2-5-1": { @@ -482,6 +494,17 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", "version": "==1.1.3.post0" }, +<<<<<<< Updated upstream +======= + "gunicorn": { + "extras": [ + "eventlet" + ], + "git": "https://github.com/benoitc/gunicorn.git", + "ref": "1299ea9e967a61ae2edebe191082fd169b864c64", + "version": "==20.1.0" + }, +>>>>>>> Stashed changes "gunicorn[eventlet]==20-1-0": { "git": "https://github.com/benoitc/gunicorn.git", "ref": "1299ea9e967a61ae2edebe191082fd169b864c64" @@ -732,7 +755,11 @@ }, "notifications-utils": { "git": "https://github.com/GSA/notifications-utils.git", +<<<<<<< Updated upstream "ref": "90c12da575f4e481452d4fcd2a594204b0c28249" +======= + "ref": "2cdffe3fa2417b61ce3d714dc5a2d67de6632bdd" +>>>>>>> Stashed changes }, "orderedset": { "hashes": [ @@ -1034,7 +1061,11 @@ "sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983", "sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349" ], +<<<<<<< Updated upstream "markers": "python_version >= '3.7' and python_full_version < '4.0.0'", +======= + "markers": "python_version >= '3.7' and python_version < '4'", +>>>>>>> Stashed changes "version": "==2.28.1" }, "rfc3339-validator": { @@ -1056,7 +1087,11 @@ "sha256:78f9a9bf4e7be0c5ded4583326e7461e3a3c5aae24073648b4bdfa797d78c9d2", "sha256:9d689e6ca1b3038bc82bf8d23e944b6b6037bc02301a574935b2dd946e0353b9" ], +<<<<<<< Updated upstream "markers": "python_version >= '3.5' and python_full_version < '4.0.0'", +======= + "markers": "python_version >= '3.5' and python_version < '4'", +>>>>>>> Stashed changes "version": "==4.7.2" }, "s3transfer": { @@ -1211,7 +1246,11 @@ "sha256:3fa96cf423e6987997fc326ae8df396db2a8b7c667747d47ddd8ecba91f4a74e", "sha256:b930dd878d5a8afb066a637fbb35144fe7901e3b209d1cd4f524bd0e9deee997" ], +<<<<<<< Updated upstream "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' and python_full_version < '4.0.0'", +======= + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' and python_version < '4'", +>>>>>>> Stashed changes "version": "==1.26.12" }, "vine": { @@ -1323,11 +1362,19 @@ }, "zipp": { "hashes": [ +<<<<<<< Updated upstream "sha256:3a7af91c3db40ec72dd9d154ae18e008c69efe8ca88dde4f9a731bb82fe2f9eb", "sha256:972cfa31bc2fedd3fa838a51e9bc7e64b7fb725a8c00e7431554311f180e9980" ], "markers": "python_version >= '3.7'", "version": "==3.9.0" +======= + "sha256:4fcb6f278987a6605757302a6e40e896257570d11c51628968ccb2a47e80c6c1", + "sha256:7a7262fd930bd3e36c50b9a64897aec3fafff3dfdeec9623ae22b40e93f99bb8" + ], + "markers": "python_version >= '3.7'", + "version": "==3.10.0" +>>>>>>> Stashed changes } }, "develop": { @@ -1817,7 +1864,11 @@ "sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983", "sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349" ], +<<<<<<< Updated upstream "markers": "python_version >= '3.7' and python_full_version < '4.0.0'", +======= + "markers": "python_version >= '3.7' and python_version < '4'", +>>>>>>> Stashed changes "version": "==2.28.1" }, "requests-mock": { @@ -1880,7 +1931,11 @@ "sha256:3fa96cf423e6987997fc326ae8df396db2a8b7c667747d47ddd8ecba91f4a74e", "sha256:b930dd878d5a8afb066a637fbb35144fe7901e3b209d1cd4f524bd0e9deee997" ], +<<<<<<< Updated upstream "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' and python_full_version < '4.0.0'", +======= + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' and python_version < '4'", +>>>>>>> Stashed changes "version": "==1.26.12" }, "werkzeug": { diff --git a/README.md b/README.md index cb0a95a21..1bd8ed0c4 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ This repo contains: - A public-facing REST API for Notify, which teams can integrate with using [API clients built by UK](https://www.notifications.service.gov.uk/documentation) - An internal-only REST API built using Flask to manage services, users, templates, etc., which the [admin UI](http://github.com/18F/notifications-admin) talks to) -- Asynchronous workers built using Celery to put things on queues and read them off to be processed, sent to providers, updated, etc +- Asynchronous workers built using Celery to put things on queues and read them off to be processed, sent to providers, updated, etc. ## Local setup @@ -17,7 +17,7 @@ This repo contains: 1. Install dependencies into a virtual environment ``` - pipenv install --with dev + pipenv install --dev createdb notification_api flask db upgrade ``` @@ -46,7 +46,12 @@ This repo contains: If you're working in VS Code, you can also leverage Docker for a containerized dev environment -1. Create .env file as described in the .env section below. +1. Create the .env file + + ``` + cp sample.env .env + # follow the instructions in .env + ``` 1. Install the Remote-Containers plug-in in VS Code @@ -56,7 +61,7 @@ If you're working in VS Code, you can also leverage Docker for a containerized d 1. Using the command palette (shift+cmd+p) or green button thingy in the bottom left, search and select “Remote Containers: Open Folder in Container...” When prompted, choose **devcontainer-api** folder (note: this is a *subfolder* of notification-api). This will startup the container in a new window, replacing the current one. -1. Wait a few minutes while things happen +1. Wait a few minutes while things happen 🍵 1. Open a VS Code terminal and run the Flask application: diff --git a/requirements.in b/requirements.in deleted file mode 100644 index dbcc0f8a6..000000000 --- a/requirements.in +++ /dev/null @@ -1,39 +0,0 @@ -# Run `make freeze-requirements` to update requirements.txt -# with package version changes made in requirements.in - -cffi==1.15.0 -celery[redis]==5.2.7 -Flask-Bcrypt==1.0.1 -flask-marshmallow==0.14.0 -Flask-Migrate==3.1.0 -git+https://github.com/pallets-eco/flask-sqlalchemy.git@aa7a61a5357cf6f5dcc135d98c781192457aa6fa#egg=Flask-SQLAlchemy==2.5.1 -Flask==2.1.2 -click-datetime==0.2 -# Should be pinned until a new gunicorn release greater than 20.1.0 comes out. (Due to eventlet v0.33 compatibility issues) -git+https://github.com/benoitc/gunicorn.git@1299ea9e967a61ae2edebe191082fd169b864c64#egg=gunicorn[eventlet]==20.1.0 -iso8601==1.0.2 -itsdangerous==2.1.2 -jsonschema[format]==4.5.1 -marshmallow-sqlalchemy==0.28.1 -marshmallow==3.15.0 -psycopg2-binary==2.9.3 -PyJWT==2.4.0 -SQLAlchemy==1.4.40 -cachetools==5.1.0 -beautifulsoup4==4.11.1 -lxml==4.9.1 -defusedxml==0.7.1 -Werkzeug==2.1.1 -python-dotenv==0.20.0 -oscrypto==1.3.0 - -notifications-python-client==6.3.0 - -# PaaS -awscli-cwlogs==1.4.6 - -notifications-utils @ git+https://github.com/GSA/notifications-utils.git#egg=notifications-utils - -# gds-metrics requires prometheseus 0.2.0, override that requirement as 0.7.1 brings significant performance gains -prometheus-client==0.14.1 -git+https://github.com/alphagov/gds_metrics_python.git@6f1840a57b6fb1ee40b7e84f2f18ec229de8aa72#egg=gds-metrics diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index c5e41644e..000000000 --- a/requirements.txt +++ /dev/null @@ -1,297 +0,0 @@ -# -# This file is autogenerated by pip-compile with python 3.9 -# To update, run: -# -# pip-compile requirements.in -# -alembic==1.7.7 - # via flask-migrate -amqp==5.1.1 - # via kombu -arrow==1.2.2 - # via isoduration -asn1crypto==1.5.1 - # via oscrypto -async-timeout==4.0.2 - # via redis -attrs==21.4.0 - # via jsonschema -awscli==1.24.8 - # via awscli-cwlogs -awscli-cwlogs==1.4.6 - # via -r requirements.in -bcrypt==3.2.2 - # via flask-bcrypt -beautifulsoup4==4.11.1 - # via -r requirements.in -billiard==3.6.4.0 - # via celery -bleach==4.1.0 - # via notifications-utils -blinker==1.4 - # via gds-metrics -boto3==1.23.8 - # via notifications-utils -botocore==1.26.8 - # via - # awscli - # boto3 - # s3transfer -cachetools==5.1.0 - # via - # -r requirements.in - # notifications-utils -celery[redis]==5.2.7 - # via -r requirements.in -certifi==2022.5.18.1 - # via - # pyproj - # requests -cffi==1.15.0 - # via - # -r requirements.in - # bcrypt -charset-normalizer==2.0.12 - # via requests -click==8.1.3 - # via - # celery - # click-datetime - # click-didyoumean - # click-plugins - # click-repl - # flask -click-datetime==0.2 - # via -r requirements.in -click-didyoumean==0.3.0 - # via celery -click-plugins==1.1.1 - # via celery -click-repl==0.2.0 - # via celery -colorama==0.4.4 - # via awscli -defusedxml==0.7.1 - # via -r requirements.in -deprecated==1.2.13 - # via redis -dnspython==2.2.1 - # via eventlet -docopt==0.6.2 - # via notifications-python-client -docutils==0.16 - # via awscli -eventlet==0.33.1 - # via gunicorn -flask==2.1.2 - # via - # -r requirements.in - # flask-bcrypt - # flask-marshmallow - # flask-migrate - # flask-redis - # flask-sqlalchemy - # gds-metrics - # notifications-utils -flask-bcrypt==1.0.1 - # via -r requirements.in -flask-marshmallow==0.14.0 - # via -r requirements.in -flask-migrate==3.1.0 - # via -r requirements.in -flask-redis==0.4.0 - # via notifications-utils -flask-sqlalchemy @ git+https://github.com/pallets-eco/flask-sqlalchemy.git@aa7a61a5357cf6f5dcc135d98c781192457aa6fa - # via - # -r requirements.in - # flask-migrate -fqdn==1.5.1 - # via jsonschema -gds-metrics @ git+https://github.com/alphagov/gds_metrics_python.git@6f1840a57b6fb1ee40b7e84f2f18ec229de8aa72 - # via -r requirements.in -geojson==2.5.0 - # via notifications-utils -govuk-bank-holidays==0.11 - # via notifications-utils -greenlet==1.1.2 - # via - # eventlet - # sqlalchemy -gunicorn @ git+https://github.com/benoitc/gunicorn.git@1299ea9e967a61ae2edebe191082fd169b864c64 - # via -r requirements.in -idna==3.3 - # via - # jsonschema - # requests -importlib-metadata==4.12.0 - # via flask -iso8601==1.0.2 - # via -r requirements.in -isoduration==20.11.0 - # via jsonschema -itsdangerous==2.1.2 - # via - # -r requirements.in - # flask - # notifications-utils -jinja2==3.1.2 - # via - # flask - # notifications-utils -jmespath==1.0.0 - # via - # boto3 - # botocore -jsonpointer==2.3 - # via jsonschema -jsonschema[format]==4.5.1 - # via -r requirements.in -kombu==5.2.4 - # via celery -lxml==4.9.1 - # via -r requirements.in -mako==1.2.2 - # via alembic -markupsafe==2.1.1 - # via - # jinja2 - # mako -marshmallow==3.15.0 - # via - # -r requirements.in - # flask-marshmallow - # marshmallow-sqlalchemy -marshmallow-sqlalchemy==0.28.1 - # via -r requirements.in -mistune==0.8.4 - # via notifications-utils -notifications-python-client==6.3.0 - # via -r requirements.in -notifications-utils @ git+https://github.com/GSA/notifications-utils.git - # via -r requirements.in -orderedset==2.0.3 - # via notifications-utils -oscrypto==1.3.0 - # via -r requirements.in -packaging==21.3 - # via - # bleach - # marshmallow - # marshmallow-sqlalchemy - # redis -phonenumbers==8.12.48 - # via notifications-utils -prometheus-client==0.14.1 - # via - # -r requirements.in - # gds-metrics -prompt-toolkit==3.0.29 - # via click-repl -psycopg2-binary==2.9.3 - # via -r requirements.in -pyasn1==0.4.8 - # via rsa -pycparser==2.21 - # via cffi -pyjwt==2.4.0 - # via - # -r requirements.in - # notifications-python-client -pyparsing==3.0.9 - # via packaging -pypdf2==2.0.0 - # via notifications-utils -pyproj==3.3.1 - # via notifications-utils -pyrsistent==0.18.1 - # via jsonschema -python-dateutil==2.8.2 - # via - # arrow - # awscli-cwlogs - # botocore -python-dotenv==0.20.0 - # via -r requirements.in -python-json-logger==2.0.2 - # via notifications-utils -pytz==2022.1 - # via - # celery - # notifications-utils -pyyaml==5.4.1 - # via - # awscli - # notifications-utils -redis==4.3.1 - # via - # celery - # flask-redis -requests==2.27.1 - # via - # awscli-cwlogs - # govuk-bank-holidays - # notifications-python-client - # notifications-utils -rfc3339-validator==0.1.4 - # via jsonschema -rfc3987==1.3.8 - # via jsonschema -rsa==4.7.2 - # via awscli -s3transfer==0.5.2 - # via - # awscli - # boto3 -shapely==1.8.2 - # via notifications-utils -six==1.16.0 - # via - # awscli-cwlogs - # bleach - # click-repl - # eventlet - # flask-marshmallow - # python-dateutil - # rfc3339-validator -smartypants==2.0.1 - # via notifications-utils -soupsieve==2.3.2.post1 - # via beautifulsoup4 -sqlalchemy==1.4.40 - # via - # -r requirements.in - # alembic - # flask-sqlalchemy - # marshmallow-sqlalchemy -statsd==3.3.0 - # via notifications-utils -typing-extensions==4.3.0 - # via pypdf2 -uri-template==1.2.0 - # via jsonschema -urllib3==1.26.9 - # via - # botocore - # requests -vine==5.0.0 - # via - # amqp - # celery - # kombu -wcwidth==0.2.5 - # via prompt-toolkit -webcolors==1.12 - # via jsonschema -webencodings==0.5.1 - # via bleach -werkzeug==2.1.1 - # via - # -r requirements.in - # flask -wrapt==1.14.1 - # via deprecated -zipp==3.8.1 - # via importlib-metadata - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/requirements_for_test.txt b/requirements_for_test.txt deleted file mode 100644 index 3c6756bc1..000000000 --- a/requirements_for_test.txt +++ /dev/null @@ -1,14 +0,0 @@ ---requirement requirements.txt -flake8==4.0.1 -flake8-bugbear==22.4.25 -isort==5.10.1 -moto==3.1.9 -pytest==7.1.2 -pytest-env==0.6.2 -pytest-mock==3.7.0 -pytest-cov==3.0.0 -pytest-xdist==2.5.0 -freezegun==1.2.1 -requests-mock==1.9.3 -# used for creating manifest file locally -jinja2-cli[yaml]==0.8.2