From 2ed50e760fc5bcf5d5ddb80710ae2d2b9e45d237 Mon Sep 17 00:00:00 2001 From: Leo Hemsted Date: Tue, 9 Oct 2018 13:27:49 +0100 Subject: [PATCH] Revert "Celery 4" --- Makefile | 33 +++++++++------------------------ app/config.py | 18 +++++++++++++----- app/dao/notifications_dao.py | 2 +- docker/Dockerfile | 1 - requirements-app.txt | 4 ++-- requirements.txt | 15 +++++++-------- scripts/bootstrap.sh | 11 ++++++++++- 7 files changed, 42 insertions(+), 42 deletions(-) diff --git a/Makefile b/Makefile index e94889cea..3612540bc 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,7 @@ SHELL := /bin/bash DATE = $(shell date +%Y-%m-%d:%H:%M:%S) +PIP_ACCEL_CACHE ?= ${CURDIR}/cache/pip-accel APP_VERSION_FILE = app/version.py GIT_BRANCH ?= $(shell git symbolic-ref --short HEAD 2> /dev/null || echo "detached") @@ -33,19 +34,11 @@ help: @cat $(MAKEFILE_LIST) | grep -E '^[a-zA-Z_-]+:.*?## .*$$' | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' .PHONY: venv -# if there's a virtualenv already active, we don't want to do anything -ifeq ($(VIRTUAL_ENV),) venv: venv/bin/activate ## Create virtualenv if it does not exist venv/bin/activate: test -d venv || virtualenv venv -p python3 - -VENV_BIN_DIR = $(shell . venv/bin/activate && echo $$VIRTUAL_ENV/bin) -else -venv: - -VENV_BIN_DIR = $(VIRTUAL_ENV)/bin -endif + . venv/bin/activate && pip install pip-accel .PHONY: check-env-vars check-env-vars: ## Check mandatory environment variables @@ -72,9 +65,9 @@ production: ## Set environment to production @true .PHONY: dependencies -dependencies: venv - $(call install-pycurl, $(VENV_BIN_DIR)/pip) - $(VENV_BIN_DIR)/pip install -r requirements_for_test.txt +dependencies: venv ## Install build dependencies + mkdir -p ${PIP_ACCEL_CACHE} + . venv/bin/activate && PIP_ACCEL_CACHE=${PIP_ACCEL_CACHE} pip-accel install -r requirements_for_test.txt .PHONY: generate-version-file generate-version-file: ## Generates the app version file @@ -82,7 +75,7 @@ generate-version-file: ## Generates the app version file .PHONY: build build: dependencies generate-version-file ## Build project - + . venv/bin/activate && PIP_ACCEL_CACHE=${PIP_ACCEL_CACHE} pip-accel install -r requirements.txt .PHONY: build-paas-artifact build-paas-artifact: ## Build the deploy artifact for PaaS @@ -104,7 +97,6 @@ test: venv generate-version-file ## Run tests freeze-requirements: rm -rf venv-freeze virtualenv -p python3 venv-freeze - $(call install-pycurl, $$(pwd)/venv-freeze/bin/pip) $$(pwd)/venv-freeze/bin/pip install -r requirements-app.txt echo '# pyup: ignore file' > requirements.txt echo '# This file is autogenerated. Do not edit it manually.' >> requirements.txt @@ -113,15 +105,6 @@ freeze-requirements: $$(pwd)/venv-freeze/bin/pip freeze -r <(sed '/^--/d' requirements-app.txt) | sed -n '/The following requirements were added by pip freeze/,$$p' >> requirements.txt rm -rf venv-freeze -define install-pycurl - # install pycurl separately to avoid flags disabling wheels for other packages - PYCURL_SSL_LIBRARY=openssl ${1} install pycurl==7.43.0.2 --global-option="build_ext" --global-option="-I/usr/local/opt/openssl/include" -endef - -.PHONY: install-pycurl -install-pycurl: - $(call install-pycurl, pip) - .PHONY: test-requirements test-requirements: @diff requirements-app.txt requirements.txt | grep '<' \ @@ -131,10 +114,11 @@ test-requirements: .PHONY: coverage coverage: venv ## Create coverage report - coveralls + . venv/bin/activate && coveralls .PHONY: prepare-docker-build-image prepare-docker-build-image: ## Prepare the Docker builder image + mkdir -p ${PIP_ACCEL_CACHE} make -C docker build .PHONY: build-with-docker @@ -142,6 +126,7 @@ build-with-docker: prepare-docker-build-image ## Build inside a Docker container @docker run -i${DOCKER_TTY} --rm \ --name "${DOCKER_CONTAINER_PREFIX}-build" \ -v "`pwd`:/var/project" \ + -v "${PIP_ACCEL_CACHE}:/var/project/cache/pip-accel" \ -e UID=$(shell id -u) \ -e GID=$(shell id -g) \ -e GIT_COMMIT=${GIT_COMMIT} \ diff --git a/app/config.py b/app/config.py index c37dd5d00..ae7788d34 100644 --- a/app/config.py +++ b/app/config.py @@ -150,9 +150,9 @@ class Config(object): BROKER_URL = 'sqs://' BROKER_TRANSPORT_OPTIONS = { 'region': AWS_REGION, + 'polling_interval': 1, # 1 second 'visibility_timeout': 310, - 'queue_name_prefix': NOTIFICATION_QUEUE_PREFIX, - 'wait_time_seconds': 20 # enable long polling, with a wait time of 20 seconds + 'queue_name_prefix': NOTIFICATION_QUEUE_PREFIX } CELERY_ENABLE_UTC = True CELERY_TIMEZONE = 'Europe/London' @@ -270,9 +270,7 @@ class Config(object): 'options': {'queue': QueueNames.PERIODIC} } } - - # this is overriden by the -Q command, but locally, we should read from all queues - CELERY_QUEUES = [Queue(queue, Exchange('default'), routing_key=queue) for queue in QueueNames.all_queues()] + CELERY_QUEUES = [] NOTIFICATIONS_ALERT = 5 # five mins FROM_NUMBER = 'development' @@ -360,6 +358,11 @@ class Development(Config): STATSD_PORT = 1000 STATSD_PREFIX = "stats-prefix" + for queue in QueueNames.all_queues(): + Config.CELERY_QUEUES.append( + Queue(queue, Exchange('default'), routing_key=queue) + ) + API_HOST_NAME = "http://localhost:6011" API_RATE_LIMIT_ENABLED = True @@ -382,6 +385,11 @@ class Test(Development): BROKER_URL = 'you-forgot-to-mock-celery-in-your-tests://' + for queue in QueueNames.all_queues(): + Config.CELERY_QUEUES.append( + Queue(queue, Exchange('default'), routing_key=queue) + ) + API_RATE_LIMIT_ENABLED = True API_HOST_NAME = "http://localhost:6011" diff --git a/app/dao/notifications_dao.py b/app/dao/notifications_dao.py index 6cfca54a8..95896e4da 100644 --- a/app/dao/notifications_dao.py +++ b/app/dao/notifications_dao.py @@ -5,7 +5,7 @@ from datetime import ( timedelta, ) -from botocore.exceptions import ClientError as BotoClientError +from boto.exception import BotoClientError from flask import current_app from notifications_utils.recipients import ( diff --git a/docker/Dockerfile b/docker/Dockerfile index db1bc6202..1060824cf 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,6 @@ RUN \ libffi-dev \ python-dev \ jq \ - libcurl4-openssl-dev \ && echo "Clean up" \ && rm -rf /var/lib/apt/lists/* /tmp/* diff --git a/requirements-app.txt b/requirements-app.txt index 085306c17..5aeebe34f 100644 --- a/requirements-app.txt +++ b/requirements-app.txt @@ -2,7 +2,7 @@ # with package version changes made in requirements-app.txt cffi==1.11.5 -celery[sqs]==4.2.1 +celery==3.1.26.post2 # pyup: <4 docopt==0.6.2 Flask-Bcrypt==0.7.1 flask-marshmallow==0.9.0 @@ -29,4 +29,4 @@ botocore<1.11.0 git+https://github.com/alphagov/notifications-utils.git@30.5.3#egg=notifications-utils==30.5.3 -# if you want to update pycurl please do so in makefile +git+https://github.com/alphagov/boto.git@2.43.0-patch3#egg=boto==2.43.0-patch3 diff --git a/requirements.txt b/requirements.txt index 89d555659..4a8422f37 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ # with package version changes made in requirements-app.txt cffi==1.11.5 -celery[sqs]==4.2.1 +celery==3.1.26.post2 # pyup: <4 docopt==0.6.2 Flask-Bcrypt==0.7.1 flask-marshmallow==0.9.0 @@ -31,15 +31,16 @@ botocore<1.11.0 git+https://github.com/alphagov/notifications-utils.git@30.5.3#egg=notifications-utils==30.5.3 -# if you want to update pycurl please do so in makefile +git+https://github.com/alphagov/boto.git@2.43.0-patch3#egg=boto==2.43.0-patch3 ## The following requirements were added by pip freeze: alembic==1.0.0 -amqp==2.3.2 +amqp==1.4.9 +anyjson==0.3.3 bcrypt==3.1.4 -billiard==3.5.0.4 +billiard==3.3.0.23 bleach==2.1.3 -boto3==1.9.17 +boto3==1.6.16 certifi==2018.8.24 chardet==3.0.4 Click==7.0 @@ -53,7 +54,7 @@ idna==2.7 itsdangerous==0.24 Jinja2==2.10 jmespath==0.9.3 -kombu==4.2.1 +kombu==3.0.37 Mako==1.0.7 MarkupSafe==1.0 mistune==0.8.3 @@ -62,7 +63,6 @@ orderedset==2.0.1 phonenumbers==8.9.4 pyasn1==0.4.4 pycparser==2.19 -pycurl==7.43.0.2 PyPDF2==1.26.0 python-dateutil==2.7.3 python-editor==1.0.3 @@ -77,6 +77,5 @@ six==1.11.0 smartypants==2.0.1 statsd==3.2.2 urllib3==1.23 -vine==1.1.4 webencodings==0.5.1 Werkzeug==0.14.1 diff --git a/scripts/bootstrap.sh b/scripts/bootstrap.sh index b92832233..c46d03aa6 100755 --- a/scripts/bootstrap.sh +++ b/scripts/bootstrap.sh @@ -20,7 +20,16 @@ function display_result { fi } -make build +if [ ! $VIRTUAL_ENV ]; then + virtualenv -p python3 ./venv + . ./venv/bin/activate +fi + +# we need the version file to exist otherwise the app will blow up +make generate-version-file + +# Install Python development dependencies +pip3 install -r requirements_for_test.txt # Create Postgres databases createdb notification_api