2016-08-16 17:51:52 +01:00
|
|
|
.DEFAULT_GOAL := help
|
|
|
|
|
SHELL := /bin/bash
|
|
|
|
|
DATE = $(shell date +%Y-%m-%d:%H:%M:%S)
|
|
|
|
|
|
|
|
|
|
APP_VERSION_FILE = app/version.py
|
|
|
|
|
|
2016-08-26 16:18:04 +01:00
|
|
|
GIT_BRANCH ?= $(shell git symbolic-ref --short HEAD 2> /dev/null || echo "detached")
|
2016-08-16 17:51:52 +01:00
|
|
|
GIT_COMMIT ?= $(shell git rev-parse HEAD)
|
|
|
|
|
|
2016-12-08 12:12:45 +00:00
|
|
|
CF_API ?= api.cloud.service.gov.uk
|
|
|
|
|
CF_ORG ?= govuk-notify
|
|
|
|
|
CF_SPACE ?= ${DEPLOY_ENV}
|
2017-02-28 12:32:01 +00:00
|
|
|
CF_HOME ?= ${HOME}
|
|
|
|
|
$(eval export CF_HOME)
|
2016-12-08 12:12:45 +00:00
|
|
|
|
2020-12-07 17:51:15 +00:00
|
|
|
CF_MANIFEST_PATH ?= /tmp/manifest.yml
|
|
|
|
|
|
2017-02-13 18:04:08 +00:00
|
|
|
|
2018-01-05 14:32:20 +00:00
|
|
|
NOTIFY_CREDENTIALS ?= ~/.notify-credentials
|
|
|
|
|
|
2020-03-03 11:51:50 +00:00
|
|
|
|
|
|
|
|
## DEVELOPMENT
|
|
|
|
|
|
2021-02-17 17:22:45 +00:00
|
|
|
.PHONY: bootstrap
|
2021-04-27 12:00:53 +01:00
|
|
|
bootstrap: generate-version-file ## Set up everything to run the app
|
2021-02-17 17:22:45 +00:00
|
|
|
pip3 install -r requirements_for_test.txt
|
2021-02-18 15:19:40 +00:00
|
|
|
createdb notification_api || true
|
2021-06-15 17:33:19 +01:00
|
|
|
(. environment.sh && flask db upgrade) || true
|
2021-02-17 17:22:45 +00:00
|
|
|
|
2021-02-17 16:49:05 +00:00
|
|
|
.PHONY: run-flask
|
2021-04-27 12:00:53 +01:00
|
|
|
run-flask: ## Run flask
|
2021-02-17 16:49:05 +00:00
|
|
|
. environment.sh && flask run -p 6011
|
|
|
|
|
|
|
|
|
|
.PHONY: run-celery
|
2021-04-27 12:00:53 +01:00
|
|
|
run-celery: ## Run celery
|
2021-02-17 16:49:05 +00:00
|
|
|
. environment.sh && celery \
|
|
|
|
|
-A run_celery.notify_celery worker \
|
|
|
|
|
--pidfile="/tmp/celery.pid" \
|
|
|
|
|
--loglevel=INFO \
|
Investigate duplicate tasks
This proves that a task can be duplicated **even** when the prefetch
multiplier [1] is set to 1. This is because prefetched tasks are still
present on the SQS queue and subject to the visibility timeout, whereas
tasks in progress are not (deleted, unless acks_late is set [2]).
I get the same results with:
- A concurrency of 2 processes; or
- A concurrency of 1 and 2 workers
This is consistent with [3], which reinforces that there's no way to
prevent this behaviour for successive, long-running tasks. Arguably the
current behaviour - without acks_late - is worse because the task that
gets duplicated is not the task at "fault" (the long-running one).
Suggested actions
-----------------
Keep the worker as-is: the settings are appropriate for long-running
tasks and do help minimise any duplication. Turning on acks_late has a
risk of creating new kinds of duplication - may be worse than now.
Continue other work to parallelise and optimise long-running tasks, so
that the visibility timeout is never an issue in the first place.
Demo of task duplication
------------------------
In order to do the demo we need:
- 3 tasks and 2 proceses: the hypothesis is that process #1 will pick
up tasks #1 and #2, with task #2 "timing out".
- To poll more often than the timeout, so that process #2 will pick
up (duplicate) task #2 after it finishes #3.
Alternatively, use two workers:
source environment.sh && celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=1
source environment.sh && celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=1
Command to schedule the tasks:
from app.celery.reporting_tasks import long_task
[long_task.apply_async(queue='reporting-tasks') for i in [1,2,3]]
Logs that show the duplication:
[2022-01-12 16:50:09,333: INFO/MainProcess] Task long[4c23eaf0-69fc-4951-b215-437d79e30462] received
[2022-01-12 16:50:09,901: INFO/ForkPoolWorker-1] Running long_task
[2022-01-12 16:50:10,208: INFO/MainProcess] Task long[7c22c30d-e61d-4d2d-834c-7dae56b20064] received
[2022-01-12 16:50:10,253: INFO/ForkPoolWorker-2] Running long_task
[2022-01-12 16:50:11,373: INFO/MainProcess] Task long[67e5526d-4a82-430b-8eb5-b8022a20bf70] received
[2022-01-12 16:50:21,410: INFO/MainProcess] Task long[67e5526d-4a82-430b-8eb5-b8022a20bf70] received
[2022-01-12 16:50:30,499: INFO/ForkPoolWorker-3] Running long_task
[2022-01-12 16:50:36,555: INFO/ForkPoolWorker-4] Running long_task
Note that when the first two tasks are received varies: I've seen them
received separately and at the same time.
[1]: https://docs.celeryproject.org/en/stable/userguide/configuration.html#std-setting-worker_prefetch_multiplier
[2]: https://docs.celeryproject.org/en/stable/userguide/tasks.html#Task.acks_late
[3]: https://github.com/celery/celery/issues/2788#issuecomment-136273421
2022-01-12 16:18:53 +00:00
|
|
|
--concurrency=2
|
2021-02-17 16:49:05 +00:00
|
|
|
|
|
|
|
|
.PHONY: run-celery-beat
|
2021-04-27 12:00:53 +01:00
|
|
|
run-celery-beat: ## Run celery beat
|
2021-02-17 16:49:05 +00:00
|
|
|
. environment.sh && celery \
|
|
|
|
|
-A run_celery.notify_celery beat \
|
|
|
|
|
--loglevel=INFO
|
|
|
|
|
|
2016-08-16 17:51:52 +01:00
|
|
|
.PHONY: help
|
|
|
|
|
help:
|
|
|
|
|
@cat $(MAKEFILE_LIST) | grep -E '^[a-zA-Z_-]+:.*?## .*$$' | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
|
|
|
|
|
|
|
|
|
|
.PHONY: generate-version-file
|
|
|
|
|
generate-version-file: ## Generates the app version file
|
2020-04-21 15:57:01 +01:00
|
|
|
@echo -e "__git_commit__ = \"${GIT_COMMIT}\"\n__time__ = \"${DATE}\"" > ${APP_VERSION_FILE}
|
2016-08-16 17:51:52 +01:00
|
|
|
|
|
|
|
|
.PHONY: test
|
2021-11-10 14:05:46 +00:00
|
|
|
test: ## Run tests
|
2021-06-16 13:01:56 +01:00
|
|
|
flake8 .
|
|
|
|
|
isort --check-only ./app ./tests
|
|
|
|
|
pytest -n4 --maxfail=10
|
2016-08-16 17:51:52 +01:00
|
|
|
|
Pin all application requirements in requirements.txt
The list of top-level dependencies is moved to requirements-app.txt,
which is used by `make freeze-requirements` to generate the full
list of requirements in requirements.txt.
This is based on alphagov/digitalmarketplace-api#615, so rationale
from that PR applies here.
We had a problem with unpinned packages on new deployments leading
to failed tests (e.g. alphagov/notifications-admin#2144) which is
why we're implementing this now.
After re-evaluating pipenv again, this still seems like the least
disruptive approach:
* pyup.io has experimental support for Pipfile, but doesn't respect
version ranges or updating hashes in the lock file
* CloudFoundry buildpack recognizes and supports Pipfiles out of the
box, but the support is relatively new. For example until recently
CF would install dev packages during deployment. It's also based on
generating a requirements file from the Pipfile, which doesn't
properly support pinning VCS dependencies (eg it doesn't set the
#egg= version, meaning pip will not upgrade the package if it's
already installed).
* pipenv has a strict dependency resolution algorithm, which doesn't
appear to be well documented and can cause some unexpected failures.
For example, pipenv doesn't seem to be able to install `awscli-cwlogs`
package at all, believing it to have a version conflict for `botocore`
(which it doesn't list as a direct dependency) while neither `pip` nor
`pip-tools` highlight any issues with it.
* While trying out `pipenv install` on our list of dependencies it would
regularly fail to install utils with a "Will try again." message.
While the installation succeeds after a retry, this doesn't inspire
confidence.
* The switch to Pipfile and pipenv-managed virtualenvs requires a series
of changes to `make` targets and scripts - replacing `pip install` with
`pipenv`, removing references to requirements files and prefixing
commands with `pipenv run`. While it's likely to simplify the overall
process of managing dependencies, it would require time to properly
implement across our applications and environments (Jenkins, PaaS,
docker containers, and dev machines).
2018-07-10 14:50:30 +01:00
|
|
|
.PHONY: freeze-requirements
|
2019-10-28 11:26:55 +00:00
|
|
|
freeze-requirements: ## Pin all requirements including sub dependencies into requirements.txt
|
2021-11-10 14:05:46 +00:00
|
|
|
pip install --upgrade pip-tools
|
|
|
|
|
pip-compile requirements.in
|
Pin all application requirements in requirements.txt
The list of top-level dependencies is moved to requirements-app.txt,
which is used by `make freeze-requirements` to generate the full
list of requirements in requirements.txt.
This is based on alphagov/digitalmarketplace-api#615, so rationale
from that PR applies here.
We had a problem with unpinned packages on new deployments leading
to failed tests (e.g. alphagov/notifications-admin#2144) which is
why we're implementing this now.
After re-evaluating pipenv again, this still seems like the least
disruptive approach:
* pyup.io has experimental support for Pipfile, but doesn't respect
version ranges or updating hashes in the lock file
* CloudFoundry buildpack recognizes and supports Pipfiles out of the
box, but the support is relatively new. For example until recently
CF would install dev packages during deployment. It's also based on
generating a requirements file from the Pipfile, which doesn't
properly support pinning VCS dependencies (eg it doesn't set the
#egg= version, meaning pip will not upgrade the package if it's
already installed).
* pipenv has a strict dependency resolution algorithm, which doesn't
appear to be well documented and can cause some unexpected failures.
For example, pipenv doesn't seem to be able to install `awscli-cwlogs`
package at all, believing it to have a version conflict for `botocore`
(which it doesn't list as a direct dependency) while neither `pip` nor
`pip-tools` highlight any issues with it.
* While trying out `pipenv install` on our list of dependencies it would
regularly fail to install utils with a "Will try again." message.
While the installation succeeds after a retry, this doesn't inspire
confidence.
* The switch to Pipfile and pipenv-managed virtualenvs requires a series
of changes to `make` targets and scripts - replacing `pip install` with
`pipenv`, removing references to requirements files and prefixing
commands with `pipenv run`. While it's likely to simplify the overall
process of managing dependencies, it would require time to properly
implement across our applications and environments (Jenkins, PaaS,
docker containers, and dev machines).
2018-07-10 14:50:30 +01:00
|
|
|
|
2016-12-08 12:12:45 +00:00
|
|
|
.PHONY: clean
|
2016-08-16 17:51:52 +01:00
|
|
|
clean:
|
2020-12-07 17:51:15 +00:00
|
|
|
rm -rf node_modules cache target venv .coverage build tests/.cache ${CF_MANIFEST_PATH}
|
2016-12-08 12:12:45 +00:00
|
|
|
|
2020-03-03 11:51:50 +00:00
|
|
|
|
|
|
|
|
## DEPLOYMENT
|
|
|
|
|
|
|
|
|
|
.PHONY: preview
|
|
|
|
|
preview: ## Set environment to preview
|
|
|
|
|
$(eval export DEPLOY_ENV=preview)
|
2020-05-11 18:53:52 +01:00
|
|
|
$(eval export DNS_NAME="notify.works")
|
2020-03-03 11:51:50 +00:00
|
|
|
@true
|
|
|
|
|
|
|
|
|
|
.PHONY: staging
|
|
|
|
|
staging: ## Set environment to staging
|
|
|
|
|
$(eval export DEPLOY_ENV=staging)
|
2020-05-11 18:53:52 +01:00
|
|
|
$(eval export DNS_NAME="staging-notify.works")
|
2020-03-03 11:51:50 +00:00
|
|
|
@true
|
|
|
|
|
|
|
|
|
|
.PHONY: production
|
|
|
|
|
production: ## Set environment to production
|
|
|
|
|
$(eval export DEPLOY_ENV=production)
|
2020-05-11 18:53:52 +01:00
|
|
|
$(eval export DNS_NAME="notifications.service.gov.uk")
|
2020-03-03 11:51:50 +00:00
|
|
|
@true
|
|
|
|
|
|
2016-12-08 12:12:45 +00:00
|
|
|
.PHONY: cf-login
|
|
|
|
|
cf-login: ## Log in to Cloud Foundry
|
|
|
|
|
$(if ${CF_USERNAME},,$(error Must specify CF_USERNAME))
|
|
|
|
|
$(if ${CF_PASSWORD},,$(error Must specify CF_PASSWORD))
|
|
|
|
|
$(if ${CF_SPACE},,$(error Must specify CF_SPACE))
|
|
|
|
|
@echo "Logging in to Cloud Foundry on ${CF_API}"
|
|
|
|
|
@cf login -a "${CF_API}" -u ${CF_USERNAME} -p "${CF_PASSWORD}" -o "${CF_ORG}" -s "${CF_SPACE}"
|
|
|
|
|
|
2019-04-10 15:15:48 +01:00
|
|
|
.PHONY: generate-manifest
|
|
|
|
|
generate-manifest:
|
|
|
|
|
$(if ${CF_APP},,$(error Must specify CF_APP))
|
|
|
|
|
$(if ${CF_SPACE},,$(error Must specify CF_SPACE))
|
|
|
|
|
$(if $(shell which gpg2), $(eval export GPG=gpg2), $(eval export GPG=gpg))
|
|
|
|
|
$(if ${GPG_PASSPHRASE_TXT}, $(eval export DECRYPT_CMD=echo -n $$$${GPG_PASSPHRASE_TXT} | ${GPG} --quiet --batch --passphrase-fd 0 --pinentry-mode loopback -d), $(eval export DECRYPT_CMD=${GPG} --quiet --batch -d))
|
|
|
|
|
|
|
|
|
|
@jinja2 --strict manifest.yml.j2 \
|
|
|
|
|
-D environment=${CF_SPACE} \
|
|
|
|
|
-D CF_APP=${CF_APP} \
|
|
|
|
|
--format=yaml \
|
|
|
|
|
<(${DECRYPT_CMD} ${NOTIFY_CREDENTIALS}/credentials/${CF_SPACE}/paas/environment-variables.gpg) 2>&1
|
|
|
|
|
|
2017-02-13 18:04:08 +00:00
|
|
|
.PHONY: cf-deploy
|
2019-05-22 13:53:05 +01:00
|
|
|
cf-deploy: ## Deploys the app to Cloud Foundry
|
2017-02-13 18:04:08 +00:00
|
|
|
$(if ${CF_SPACE},,$(error Must specify CF_SPACE))
|
|
|
|
|
$(if ${CF_APP},,$(error Must specify CF_APP))
|
2019-04-24 15:55:45 +01:00
|
|
|
cf target -o ${CF_ORG} -s ${CF_SPACE}
|
2017-02-13 18:04:08 +00:00
|
|
|
@cf app --guid ${CF_APP} || exit 1
|
2017-12-14 14:23:32 +00:00
|
|
|
|
2019-05-02 11:02:49 +01:00
|
|
|
# cancel any existing deploys to ensure we can apply manifest (if a deploy is in progress you'll see ScaleDisabledDuringDeployment)
|
2020-12-07 11:54:12 +00:00
|
|
|
cf cancel-deployment ${CF_APP} || true
|
2019-05-02 11:02:49 +01:00
|
|
|
|
2020-12-07 17:51:15 +00:00
|
|
|
# generate manifest (including secrets) and write it to CF_MANIFEST_PATH (in /tmp/)
|
|
|
|
|
make -s CF_APP=${CF_APP} generate-manifest > ${CF_MANIFEST_PATH}
|
|
|
|
|
|
2020-12-07 11:54:12 +00:00
|
|
|
# fails after 15 mins if deploy doesn't work
|
2020-12-07 17:51:15 +00:00
|
|
|
# reads manifest from CF_MANIFEST_PATH
|
|
|
|
|
CF_STARTUP_TIMEOUT=15 cf push ${CF_APP} --strategy=rolling -f ${CF_MANIFEST_PATH}
|
|
|
|
|
# delete old manifest file
|
|
|
|
|
rm ${CF_MANIFEST_PATH}
|
2016-12-08 12:12:45 +00:00
|
|
|
|
|
|
|
|
.PHONY: cf-deploy-api-db-migration
|
2017-02-13 18:04:08 +00:00
|
|
|
cf-deploy-api-db-migration:
|
|
|
|
|
$(if ${CF_SPACE},,$(error Must specify CF_SPACE))
|
2019-04-24 15:55:45 +01:00
|
|
|
cf target -o ${CF_ORG} -s ${CF_SPACE}
|
2020-12-07 17:51:15 +00:00
|
|
|
make -s CF_APP=notify-api-db-migration generate-manifest > ${CF_MANIFEST_PATH}
|
|
|
|
|
|
|
|
|
|
cf push notify-api-db-migration --no-route -f ${CF_MANIFEST_PATH}
|
|
|
|
|
rm ${CF_MANIFEST_PATH}
|
|
|
|
|
|
2020-12-07 11:54:12 +00:00
|
|
|
cf run-task notify-api-db-migration --command="flask db upgrade" --name api_db_migration
|
2016-12-08 12:12:45 +00:00
|
|
|
|
2017-02-28 12:32:01 +00:00
|
|
|
.PHONY: cf-check-api-db-migration-task
|
|
|
|
|
cf-check-api-db-migration-task: ## Get the status for the last notify-api-db-migration task
|
|
|
|
|
@cf curl /v3/apps/`cf app --guid notify-api-db-migration`/tasks?order_by=-created_at | jq -r ".resources[0].state"
|
|
|
|
|
|
2017-02-13 18:04:08 +00:00
|
|
|
.PHONY: cf-rollback
|
|
|
|
|
cf-rollback: ## Rollbacks the app to the previous release
|
2016-12-08 12:12:45 +00:00
|
|
|
$(if ${CF_APP},,$(error Must specify CF_APP))
|
2020-12-07 17:51:15 +00:00
|
|
|
rm ${CF_MANIFEST_PATH}
|
2020-12-07 11:54:12 +00:00
|
|
|
cf cancel-deployment ${CF_APP}
|
2017-02-13 18:04:08 +00:00
|
|
|
|
2017-05-26 16:44:23 +01:00
|
|
|
.PHONY: check-if-migrations-to-run
|
|
|
|
|
check-if-migrations-to-run:
|
2017-05-31 15:38:57 +01:00
|
|
|
@echo $(shell python3 scripts/check_if_new_migration.py)
|
2020-05-11 18:53:52 +01:00
|
|
|
|
|
|
|
|
.PHONY: cf-deploy-failwhale
|
2021-04-27 12:00:53 +01:00
|
|
|
cf-deploy-failwhale:
|
2020-05-11 18:53:52 +01:00
|
|
|
$(if ${CF_SPACE},,$(error Must target space, eg `make preview cf-deploy-failwhale`))
|
|
|
|
|
cd ./paas-failwhale; cf push notify-api-failwhale -f manifest.yml
|
|
|
|
|
|
|
|
|
|
.PHONY: enable-failwhale
|
|
|
|
|
enable-failwhale: ## Enable the failwhale app and disable api
|
|
|
|
|
$(if ${DNS_NAME},,$(error Must target space, eg `make preview enable-failwhale`))
|
|
|
|
|
# make sure failwhale is running first
|
|
|
|
|
cf start notify-api-failwhale
|
|
|
|
|
|
|
|
|
|
cf map-route notify-api-failwhale ${DNS_NAME} --hostname api
|
|
|
|
|
cf unmap-route notify-api ${DNS_NAME} --hostname api
|
|
|
|
|
@echo "Failwhale is enabled"
|
|
|
|
|
|
|
|
|
|
.PHONY: disable-failwhale
|
|
|
|
|
disable-failwhale: ## Disable the failwhale app and enable api
|
|
|
|
|
$(if ${DNS_NAME},,$(error Must target space, eg `make preview disable-failwhale`))
|
|
|
|
|
|
|
|
|
|
cf map-route notify-api ${DNS_NAME} --hostname api
|
|
|
|
|
cf unmap-route notify-api-failwhale ${DNS_NAME} --hostname api
|
|
|
|
|
cf stop notify-api-failwhale
|
|
|
|
|
@echo "Failwhale is disabled"
|