more pipenv transition

.github/workflows/checks.yml

@@ -71,6 +71,8 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - uses: ./.github/actions/setup-project
+      - name: Create requirements.txt
+        run: pipenv requirements
       - uses: trailofbits/gh-action-pip-audit@v1.0.0
         with:
           inputs: requirements.txt

.github/workflows/daily_checks.yml

@@ -38,6 +38,8 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - uses: ./.github/actions/setup-project
+      - name: Create requirements.txt
+        run: pipenv requirements
       - uses: trailofbits/gh-action-pip-audit@v1.0.0
         with:
           inputs: requirements.txt

Makefile

@@ -7,18 +7,14 @@ APP_VERSION_FILE = app/version.py
 GIT_BRANCH ?= $(shell git symbolic-ref --short HEAD 2> /dev/null || echo "detached")
 GIT_COMMIT ?= $(shell git rev-parse HEAD)
 
-CF_SPACE ?= ${DEPLOY_ENV}
-CF_HOME ?= ${HOME}
-$(eval export CF_HOME)
-
-
 ## DEVELOPMENT
 
 .PHONY: bootstrap
-bootstrap: generate-version-file ## Set up everything to run the app
+bootstrap: ## Set up everything to run the app
-	pip3 install -r requirements_for_test.txt
+	generate-version-file
+	pipenv install ---dev
 	createdb notification_api || true
-	(flask db upgrade) || true
+	(pipenv run flask db upgrade) || true
 
 .PHONY: bootstrap-with-docker
 bootstrap-with-docker: ## Build the image to run the app in Docker
@@ -26,31 +22,23 @@ bootstrap-with-docker: ## Build the image to run the app in Docker
 
 .PHONY: run-flask
 run-flask: ## Run flask
-	flask run -p 6011 --host=0.0.0.0
+	pipenv run flask run -p 6011 --host=0.0.0.0
 
 .PHONY: run-celery
 run-celery: ## Run celery, TODO remove purge for staging/prod
-	celery -A run_celery.notify_celery purge -f
+	pipenv run celery -A run_celery.notify_celery purge -f
-	celery \
+	pipenv run celery \
 		-A run_celery.notify_celery worker \
 		--pidfile="/tmp/celery.pid" \
 		--loglevel=INFO \
 		--concurrency=4
 
-.PHONY: run-celery-with-docker
-run-celery-with-docker: ## Run celery in Docker container (useful if you can't install pycurl locally)
-	./scripts/run_with_docker.sh make run-celery
-
 .PHONY: run-celery-beat
 run-celery-beat: ## Run celery beat
-	celery \
+	pipenv run celery \
 	-A run_celery.notify_celery beat \
 	--loglevel=INFO
 
-.PHONY: run-celery-beat-with-docker
-run-celery-beat-with-docker: ## Run celery beat in Docker container (useful if you can't install pycurl locally)
-	./scripts/run_with_docker.sh make run-celery-beat
-
 .PHONY: help
 help:
 	@cat $(MAKEFILE_LIST) | grep -E '^[a-zA-Z_-]+:.*?## .*$$' | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
@@ -67,12 +55,14 @@ test: ## Run tests
 
 .PHONY: freeze-requirements
 freeze-requirements: ## Pin all requirements including sub dependencies into requirements.txt
-	pip install --upgrade pip-tools
+	pipenv lock
-	pip-compile requirements.in
+	pipenv requirements
 
 .PHONY: audit
 audit:
 	pip install --upgrade pip-audit
+	pipenv requirements > requirements.txt
+	pipenv requirements --dev > requirements_for_test.txt
 	pip-audit -r requirements.txt -l --ignore-vuln PYSEC-2022-237
 	-pip-audit -r requirements_for_test.txt -l
 
@@ -88,22 +78,6 @@ clean:
 
 ## DEPLOYMENT
 
-.PHONY: cf-login
-cf-login: ## Log in to Cloud Foundry
-	$(if ${CF_USERNAME},,$(error Must specify CF_USERNAME))
-	$(if ${CF_PASSWORD},,$(error Must specify CF_PASSWORD))
-	$(if ${CF_SPACE},,$(error Must specify CF_SPACE))
-	@echo "Logging in to Cloud Foundry on ${CF_API}"
-	@cf login -a "${CF_API}" -u ${CF_USERNAME} -p "${CF_PASSWORD}" -o "${CF_ORG}" -s "${CF_SPACE}"
-
-.PHONY: cf-check-api-db-migration-task
-cf-check-api-db-migration-task: ## Get the status for the last notifications-api task
-	@cf curl /v3/apps/`cf app --guid notifications-api`/tasks?order_by=-created_at | jq -r ".resources[0].state"
-
-.PHONY: check-if-migrations-to-run
-check-if-migrations-to-run:
-	@echo $(shell python3 scripts/check_if_new_migration.py)
-
 # .PHONY: cf-deploy-failwhale
 # cf-deploy-failwhale:
 # 	$(if ${CF_SPACE},,$(error Must target space, eg `make preview cf-deploy-failwhale`))

Pipfile.lock

@@ -1,7 +1,11 @@
 {
     "_meta": {
         "hash": {
+<<<<<<< Updated upstream
             "sha256": "eca9a39871c8db3e82fb384c39afc89fdc3c145c87653f5090927e578618ce11"
+=======
+            "sha256": "ce99b649bae4b10b084aacc936b72e01957eb9354cbc420d684590133c2da004"
+>>>>>>> Stashed changes
         },
         "pipfile-spec": 6,
         "requires": {
@@ -370,11 +374,19 @@
             "version": "==0.4.0"
         },
         "flask-sqlalchemy": {
+<<<<<<< Updated upstream
+=======
+            "git": "https://github.com/pallets-eco/flask-sqlalchemy.git",
+>>>>>>> Stashed changes
             "hashes": [
                 "sha256:2bda44b43e7cacb15d4e05ff3cc1f8bc97936cc464623424102bfc2c35e95912",
                 "sha256:f12c3d4cc5cc7fdcc148b9527ea05671718c3ea45d50c7e732cceb33f574b390"
             ],
+<<<<<<< Updated upstream
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+=======
+            "ref": "aa7a61a5357cf6f5dcc135d98c781192457aa6fa",
+>>>>>>> Stashed changes
             "version": "==2.5.1"
         },
         "flask-sqlalchemy==2-5-1": {
@@ -482,6 +494,17 @@
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
             "version": "==1.1.3.post0"
         },
+<<<<<<< Updated upstream
+=======
+        "gunicorn": {
+            "extras": [
+                "eventlet"
+            ],
+            "git": "https://github.com/benoitc/gunicorn.git",
+            "ref": "1299ea9e967a61ae2edebe191082fd169b864c64",
+            "version": "==20.1.0"
+        },
+>>>>>>> Stashed changes
         "gunicorn[eventlet]==20-1-0": {
             "git": "https://github.com/benoitc/gunicorn.git",
             "ref": "1299ea9e967a61ae2edebe191082fd169b864c64"
@@ -732,7 +755,11 @@
         },
         "notifications-utils": {
             "git": "https://github.com/GSA/notifications-utils.git",
+<<<<<<< Updated upstream
             "ref": "90c12da575f4e481452d4fcd2a594204b0c28249"
+=======
+            "ref": "2cdffe3fa2417b61ce3d714dc5a2d67de6632bdd"
+>>>>>>> Stashed changes
         },
         "orderedset": {
             "hashes": [
@@ -1034,7 +1061,11 @@
                 "sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983",
                 "sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349"
             ],
+<<<<<<< Updated upstream
             "markers": "python_version >= '3.7' and python_full_version < '4.0.0'",
+=======
+            "markers": "python_version >= '3.7' and python_version < '4'",
+>>>>>>> Stashed changes
             "version": "==2.28.1"
         },
         "rfc3339-validator": {
@@ -1056,7 +1087,11 @@
                 "sha256:78f9a9bf4e7be0c5ded4583326e7461e3a3c5aae24073648b4bdfa797d78c9d2",
                 "sha256:9d689e6ca1b3038bc82bf8d23e944b6b6037bc02301a574935b2dd946e0353b9"
             ],
+<<<<<<< Updated upstream
             "markers": "python_version >= '3.5' and python_full_version < '4.0.0'",
+=======
+            "markers": "python_version >= '3.5' and python_version < '4'",
+>>>>>>> Stashed changes
             "version": "==4.7.2"
         },
         "s3transfer": {
@@ -1211,7 +1246,11 @@
                 "sha256:3fa96cf423e6987997fc326ae8df396db2a8b7c667747d47ddd8ecba91f4a74e",
                 "sha256:b930dd878d5a8afb066a637fbb35144fe7901e3b209d1cd4f524bd0e9deee997"
             ],
+<<<<<<< Updated upstream
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' and python_full_version < '4.0.0'",
+=======
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' and python_version < '4'",
+>>>>>>> Stashed changes
             "version": "==1.26.12"
         },
         "vine": {
@@ -1323,11 +1362,19 @@
         },
         "zipp": {
             "hashes": [
+<<<<<<< Updated upstream
                 "sha256:3a7af91c3db40ec72dd9d154ae18e008c69efe8ca88dde4f9a731bb82fe2f9eb",
                 "sha256:972cfa31bc2fedd3fa838a51e9bc7e64b7fb725a8c00e7431554311f180e9980"
             ],
             "markers": "python_version >= '3.7'",
             "version": "==3.9.0"
+=======
+                "sha256:4fcb6f278987a6605757302a6e40e896257570d11c51628968ccb2a47e80c6c1",
+                "sha256:7a7262fd930bd3e36c50b9a64897aec3fafff3dfdeec9623ae22b40e93f99bb8"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==3.10.0"
+>>>>>>> Stashed changes
         }
     },
     "develop": {
@@ -1817,7 +1864,11 @@
                 "sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983",
                 "sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349"
             ],
+<<<<<<< Updated upstream
             "markers": "python_version >= '3.7' and python_full_version < '4.0.0'",
+=======
+            "markers": "python_version >= '3.7' and python_version < '4'",
+>>>>>>> Stashed changes
             "version": "==2.28.1"
         },
         "requests-mock": {
@@ -1880,7 +1931,11 @@
                 "sha256:3fa96cf423e6987997fc326ae8df396db2a8b7c667747d47ddd8ecba91f4a74e",
                 "sha256:b930dd878d5a8afb066a637fbb35144fe7901e3b209d1cd4f524bd0e9deee997"
             ],
+<<<<<<< Updated upstream
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' and python_full_version < '4.0.0'",
+=======
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' and python_version < '4'",
+>>>>>>> Stashed changes
             "version": "==1.26.12"
         },
         "werkzeug": {

README.md

@@ -6,7 +6,7 @@ This repo contains:
 
 - A public-facing REST API for Notify, which teams can integrate with using [API clients built by UK](https://www.notifications.service.gov.uk/documentation)
 - An internal-only REST API built using Flask to manage services, users, templates, etc., which the [admin UI](http://github.com/18F/notifications-admin) talks to)
-- Asynchronous workers built using Celery to put things on queues and read them off to be processed, sent to providers, updated, etc
+- Asynchronous workers built using Celery to put things on queues and read them off to be processed, sent to providers, updated, etc.
 
 ## Local setup
 
@@ -17,7 +17,7 @@ This repo contains:
 1. Install dependencies into a virtual environment
 
     ```
-    pipenv install --with dev
+    pipenv install --dev
     createdb notification_api
     flask db upgrade
     ```
@@ -46,7 +46,12 @@ This repo contains:
 
 If you're working in VS Code, you can also leverage Docker for a containerized dev environment
 
-1. Create .env file as described in the .env section below.
+1. Create the .env file
+
+    ```
+    cp sample.env .env
+    # follow the instructions in .env
+    ```
 
 1. Install the Remote-Containers plug-in in VS Code
 
@@ -56,7 +61,7 @@ If you're working in VS Code, you can also leverage Docker for a containerized d
 
 1. Using the command palette (shift+cmd+p) or green button thingy in the bottom left, search and select “Remote Containers: Open Folder in Container...” When prompted, choose **devcontainer-api** folder (note: this is a *subfolder* of notification-api). This will startup the container in a new window, replacing the current one.
 
-1. Wait a few minutes while things happen
+1. Wait a few minutes while things happen 🍵
 
 1. Open a VS Code terminal and run the Flask application:
 

requirements.in

@@ -1,39 +0,0 @@
-# Run `make freeze-requirements` to update requirements.txt
-# with package version changes made in requirements.in
-
-cffi==1.15.0
-celery[redis]==5.2.7
-Flask-Bcrypt==1.0.1
-flask-marshmallow==0.14.0
-Flask-Migrate==3.1.0
-git+https://github.com/pallets-eco/flask-sqlalchemy.git@aa7a61a5357cf6f5dcc135d98c781192457aa6fa#egg=Flask-SQLAlchemy==2.5.1
-Flask==2.1.2
-click-datetime==0.2
-# Should be pinned until a new gunicorn release greater than 20.1.0 comes out. (Due to eventlet v0.33 compatibility issues)
-git+https://github.com/benoitc/gunicorn.git@1299ea9e967a61ae2edebe191082fd169b864c64#egg=gunicorn[eventlet]==20.1.0
-iso8601==1.0.2
-itsdangerous==2.1.2
-jsonschema[format]==4.5.1
-marshmallow-sqlalchemy==0.28.1
-marshmallow==3.15.0
-psycopg2-binary==2.9.3
-PyJWT==2.4.0
-SQLAlchemy==1.4.40
-cachetools==5.1.0
-beautifulsoup4==4.11.1
-lxml==4.9.1
-defusedxml==0.7.1
-Werkzeug==2.1.1
-python-dotenv==0.20.0
-oscrypto==1.3.0
-
-notifications-python-client==6.3.0
-
-# PaaS
-awscli-cwlogs==1.4.6
-
-notifications-utils @ git+https://github.com/GSA/notifications-utils.git#egg=notifications-utils
-
-# gds-metrics requires prometheseus 0.2.0, override that requirement as 0.7.1 brings significant performance gains
-prometheus-client==0.14.1
-git+https://github.com/alphagov/gds_metrics_python.git@6f1840a57b6fb1ee40b7e84f2f18ec229de8aa72#egg=gds-metrics

requirements.txt

@@ -1,297 +0,0 @@
-#
-# This file is autogenerated by pip-compile with python 3.9
-# To update, run:
-#
-#    pip-compile requirements.in
-#
-alembic==1.7.7
-    # via flask-migrate
-amqp==5.1.1
-    # via kombu
-arrow==1.2.2
-    # via isoduration
-asn1crypto==1.5.1
-    # via oscrypto
-async-timeout==4.0.2
-    # via redis
-attrs==21.4.0
-    # via jsonschema
-awscli==1.24.8
-    # via awscli-cwlogs
-awscli-cwlogs==1.4.6
-    # via -r requirements.in
-bcrypt==3.2.2
-    # via flask-bcrypt
-beautifulsoup4==4.11.1
-    # via -r requirements.in
-billiard==3.6.4.0
-    # via celery
-bleach==4.1.0
-    # via notifications-utils
-blinker==1.4
-    # via gds-metrics
-boto3==1.23.8
-    # via notifications-utils
-botocore==1.26.8
-    # via
-    #   awscli
-    #   boto3
-    #   s3transfer
-cachetools==5.1.0
-    # via
-    #   -r requirements.in
-    #   notifications-utils
-celery[redis]==5.2.7
-    # via -r requirements.in
-certifi==2022.5.18.1
-    # via
-    #   pyproj
-    #   requests
-cffi==1.15.0
-    # via
-    #   -r requirements.in
-    #   bcrypt
-charset-normalizer==2.0.12
-    # via requests
-click==8.1.3
-    # via
-    #   celery
-    #   click-datetime
-    #   click-didyoumean
-    #   click-plugins
-    #   click-repl
-    #   flask
-click-datetime==0.2
-    # via -r requirements.in
-click-didyoumean==0.3.0
-    # via celery
-click-plugins==1.1.1
-    # via celery
-click-repl==0.2.0
-    # via celery
-colorama==0.4.4
-    # via awscli
-defusedxml==0.7.1
-    # via -r requirements.in
-deprecated==1.2.13
-    # via redis
-dnspython==2.2.1
-    # via eventlet
-docopt==0.6.2
-    # via notifications-python-client
-docutils==0.16
-    # via awscli
-eventlet==0.33.1
-    # via gunicorn
-flask==2.1.2
-    # via
-    #   -r requirements.in
-    #   flask-bcrypt
-    #   flask-marshmallow
-    #   flask-migrate
-    #   flask-redis
-    #   flask-sqlalchemy
-    #   gds-metrics
-    #   notifications-utils
-flask-bcrypt==1.0.1
-    # via -r requirements.in
-flask-marshmallow==0.14.0
-    # via -r requirements.in
-flask-migrate==3.1.0
-    # via -r requirements.in
-flask-redis==0.4.0
-    # via notifications-utils
-flask-sqlalchemy @ git+https://github.com/pallets-eco/flask-sqlalchemy.git@aa7a61a5357cf6f5dcc135d98c781192457aa6fa
-    # via
-    #   -r requirements.in
-    #   flask-migrate
-fqdn==1.5.1
-    # via jsonschema
-gds-metrics @ git+https://github.com/alphagov/gds_metrics_python.git@6f1840a57b6fb1ee40b7e84f2f18ec229de8aa72
-    # via -r requirements.in
-geojson==2.5.0
-    # via notifications-utils
-govuk-bank-holidays==0.11
-    # via notifications-utils
-greenlet==1.1.2
-    # via
-    #   eventlet
-    #   sqlalchemy
-gunicorn @ git+https://github.com/benoitc/gunicorn.git@1299ea9e967a61ae2edebe191082fd169b864c64
-    # via -r requirements.in
-idna==3.3
-    # via
-    #   jsonschema
-    #   requests
-importlib-metadata==4.12.0
-    # via flask
-iso8601==1.0.2
-    # via -r requirements.in
-isoduration==20.11.0
-    # via jsonschema
-itsdangerous==2.1.2
-    # via
-    #   -r requirements.in
-    #   flask
-    #   notifications-utils
-jinja2==3.1.2
-    # via
-    #   flask
-    #   notifications-utils
-jmespath==1.0.0
-    # via
-    #   boto3
-    #   botocore
-jsonpointer==2.3
-    # via jsonschema
-jsonschema[format]==4.5.1
-    # via -r requirements.in
-kombu==5.2.4
-    # via celery
-lxml==4.9.1
-    # via -r requirements.in
-mako==1.2.2
-    # via alembic
-markupsafe==2.1.1
-    # via
-    #   jinja2
-    #   mako
-marshmallow==3.15.0
-    # via
-    #   -r requirements.in
-    #   flask-marshmallow
-    #   marshmallow-sqlalchemy
-marshmallow-sqlalchemy==0.28.1
-    # via -r requirements.in
-mistune==0.8.4
-    # via notifications-utils
-notifications-python-client==6.3.0
-    # via -r requirements.in
-notifications-utils @ git+https://github.com/GSA/notifications-utils.git
-    # via -r requirements.in
-orderedset==2.0.3
-    # via notifications-utils
-oscrypto==1.3.0
-    # via -r requirements.in
-packaging==21.3
-    # via
-    #   bleach
-    #   marshmallow
-    #   marshmallow-sqlalchemy
-    #   redis
-phonenumbers==8.12.48
-    # via notifications-utils
-prometheus-client==0.14.1
-    # via
-    #   -r requirements.in
-    #   gds-metrics
-prompt-toolkit==3.0.29
-    # via click-repl
-psycopg2-binary==2.9.3
-    # via -r requirements.in
-pyasn1==0.4.8
-    # via rsa
-pycparser==2.21
-    # via cffi
-pyjwt==2.4.0
-    # via
-    #   -r requirements.in
-    #   notifications-python-client
-pyparsing==3.0.9
-    # via packaging
-pypdf2==2.0.0
-    # via notifications-utils
-pyproj==3.3.1
-    # via notifications-utils
-pyrsistent==0.18.1
-    # via jsonschema
-python-dateutil==2.8.2
-    # via
-    #   arrow
-    #   awscli-cwlogs
-    #   botocore
-python-dotenv==0.20.0
-    # via -r requirements.in
-python-json-logger==2.0.2
-    # via notifications-utils
-pytz==2022.1
-    # via
-    #   celery
-    #   notifications-utils
-pyyaml==5.4.1
-    # via
-    #   awscli
-    #   notifications-utils
-redis==4.3.1
-    # via
-    #   celery
-    #   flask-redis
-requests==2.27.1
-    # via
-    #   awscli-cwlogs
-    #   govuk-bank-holidays
-    #   notifications-python-client
-    #   notifications-utils
-rfc3339-validator==0.1.4
-    # via jsonschema
-rfc3987==1.3.8
-    # via jsonschema
-rsa==4.7.2
-    # via awscli
-s3transfer==0.5.2
-    # via
-    #   awscli
-    #   boto3
-shapely==1.8.2
-    # via notifications-utils
-six==1.16.0
-    # via
-    #   awscli-cwlogs
-    #   bleach
-    #   click-repl
-    #   eventlet
-    #   flask-marshmallow
-    #   python-dateutil
-    #   rfc3339-validator
-smartypants==2.0.1
-    # via notifications-utils
-soupsieve==2.3.2.post1
-    # via beautifulsoup4
-sqlalchemy==1.4.40
-    # via
-    #   -r requirements.in
-    #   alembic
-    #   flask-sqlalchemy
-    #   marshmallow-sqlalchemy
-statsd==3.3.0
-    # via notifications-utils
-typing-extensions==4.3.0
-    # via pypdf2
-uri-template==1.2.0
-    # via jsonschema
-urllib3==1.26.9
-    # via
-    #   botocore
-    #   requests
-vine==5.0.0
-    # via
-    #   amqp
-    #   celery
-    #   kombu
-wcwidth==0.2.5
-    # via prompt-toolkit
-webcolors==1.12
-    # via jsonschema
-webencodings==0.5.1
-    # via bleach
-werkzeug==2.1.1
-    # via
-    #   -r requirements.in
-    #   flask
-wrapt==1.14.1
-    # via deprecated
-zipp==3.8.1
-    # via importlib-metadata
-
-# The following packages are considered to be unsafe in a requirements file:
-# setuptools

requirements_for_test.txt

@@ -1,14 +0,0 @@
---requirement requirements.txt
-flake8==4.0.1
-flake8-bugbear==22.4.25
-isort==5.10.1
-moto==3.1.9
-pytest==7.1.2
-pytest-env==0.6.2
-pytest-mock==3.7.0
-pytest-cov==3.0.0
-pytest-xdist==2.5.0
-freezegun==1.2.1
-requests-mock==1.9.3
-# used for creating manifest file locally
-jinja2-cli[yaml]==0.8.2