Revert "Celery 4"

2025-12-10 23:32:27 -05:00 · 2018-10-09 13:27:49 +01:00
parent 72358899ad
commit 2ed50e760f
7 changed files with 42 additions and 42 deletions
--- a/33
+++ b/33
@@ -2,6 +2,7 @@
 SHELL := /bin/bash
 DATE = $(shell date +%Y-%m-%d:%H:%M:%S)

+PIP_ACCEL_CACHE ?= ${CURDIR}/cache/pip-accel
 APP_VERSION_FILE = app/version.py

 GIT_BRANCH ?= $(shell git symbolic-ref --short HEAD 2> /dev/null || echo "detached")
@@ -33,19 +34,11 @@ help:
 	@cat $(MAKEFILE_LIST) | grep -E '^[a-zA-Z_-]+:.*?## .*$$' | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'

 .PHONY: venv
-# if there's a virtualenv already active, we don't want to do anything
-ifeq ($(VIRTUAL_ENV),)
 venv: venv/bin/activate ## Create virtualenv if it does not exist

 venv/bin/activate:
 	test -d venv || virtualenv venv -p python3
-
-VENV_BIN_DIR = $(shell . venv/bin/activate && echo $$VIRTUAL_ENV/bin)
-else
-venv:
-
-VENV_BIN_DIR = $(VIRTUAL_ENV)/bin
-endif
+	. venv/bin/activate && pip install pip-accel

 .PHONY: check-env-vars
 check-env-vars: ## Check mandatory environment variables
@@ -72,9 +65,9 @@ production: ## Set environment to production
 	@true

 .PHONY: dependencies
-dependencies: venv
-	$(call install-pycurl, $(VENV_BIN_DIR)/pip)
-	$(VENV_BIN_DIR)/pip install -r requirements_for_test.txt
+dependencies: venv ## Install build dependencies
+	mkdir -p ${PIP_ACCEL_CACHE}
+	. venv/bin/activate && PIP_ACCEL_CACHE=${PIP_ACCEL_CACHE} pip-accel install -r requirements_for_test.txt

 .PHONY: generate-version-file
 generate-version-file: ## Generates the app version file
@@ -82,7 +75,7 @@ generate-version-file: ## Generates the app version file

 .PHONY: build
 build: dependencies generate-version-file ## Build project
-
+	. venv/bin/activate && PIP_ACCEL_CACHE=${PIP_ACCEL_CACHE} pip-accel install -r requirements.txt

 .PHONY: build-paas-artifact
 build-paas-artifact:  ## Build the deploy artifact for PaaS
@@ -104,7 +97,6 @@ test: venv generate-version-file ## Run tests
 freeze-requirements:
 	rm -rf venv-freeze
 	virtualenv -p python3 venv-freeze
-	$(call install-pycurl, $$(pwd)/venv-freeze/bin/pip)
 	$$(pwd)/venv-freeze/bin/pip install -r requirements-app.txt
 	echo '# pyup: ignore file' > requirements.txt
 	echo '# This file is autogenerated. Do not edit it manually.' >> requirements.txt
@@ -113,15 +105,6 @@ freeze-requirements:
 	$$(pwd)/venv-freeze/bin/pip freeze -r <(sed '/^--/d' requirements-app.txt) | sed -n '/The following requirements were added by pip freeze/,$$p' >> requirements.txt
 	rm -rf venv-freeze

-define install-pycurl
-	# install pycurl separately to avoid flags disabling wheels for other packages
-	PYCURL_SSL_LIBRARY=openssl ${1} install pycurl==7.43.0.2 --global-option="build_ext" --global-option="-I/usr/local/opt/openssl/include"
-endef
-
-.PHONY: install-pycurl
-install-pycurl:
-	$(call install-pycurl, pip)
-
 .PHONY: test-requirements
 test-requirements:
 	@diff requirements-app.txt requirements.txt | grep '<' \
@@ -131,10 +114,11 @@ test-requirements:

 .PHONY: coverage
 coverage: venv ## Create coverage report
-	coveralls
+	. venv/bin/activate && coveralls

 .PHONY: prepare-docker-build-image
 prepare-docker-build-image: ## Prepare the Docker builder image
+	mkdir -p ${PIP_ACCEL_CACHE}
 	make -C docker build

 .PHONY: build-with-docker
@@ -142,6 +126,7 @@ build-with-docker: prepare-docker-build-image ## Build inside a Docker container
 	@docker run -i${DOCKER_TTY} --rm \
 		--name "${DOCKER_CONTAINER_PREFIX}-build" \
 		-v "`pwd`:/var/project" \
+		-v "${PIP_ACCEL_CACHE}:/var/project/cache/pip-accel" \
 		-e UID=$(shell id -u) \
 		-e GID=$(shell id -g) \
 		-e GIT_COMMIT=${GIT_COMMIT} \
--- a/app/config.py
+++ b/app/config.py
@@ -150,9 +150,9 @@ class Config(object):
    BROKER_URL = 'sqs://'
    BROKER_TRANSPORT_OPTIONS = {
        'region': AWS_REGION,
+        'polling_interval': 1,  # 1 second
        'visibility_timeout': 310,
-        'queue_name_prefix': NOTIFICATION_QUEUE_PREFIX,
-        'wait_time_seconds': 20  # enable long polling, with a wait time of 20 seconds
+        'queue_name_prefix': NOTIFICATION_QUEUE_PREFIX
    }
    CELERY_ENABLE_UTC = True
    CELERY_TIMEZONE = 'Europe/London'
@@ -270,9 +270,7 @@ class Config(object):
            'options': {'queue': QueueNames.PERIODIC}
        }
    }
-
-    # this is overriden by the -Q command, but locally, we should read from all queues
-    CELERY_QUEUES = [Queue(queue, Exchange('default'), routing_key=queue) for queue in QueueNames.all_queues()]
+    CELERY_QUEUES = []

    NOTIFICATIONS_ALERT = 5  # five mins
    FROM_NUMBER = 'development'
@@ -360,6 +358,11 @@ class Development(Config):
    STATSD_PORT = 1000
    STATSD_PREFIX = "stats-prefix"

+    for queue in QueueNames.all_queues():
+        Config.CELERY_QUEUES.append(
+            Queue(queue, Exchange('default'), routing_key=queue)
+        )
+
    API_HOST_NAME = "http://localhost:6011"
    API_RATE_LIMIT_ENABLED = True

@@ -382,6 +385,11 @@ class Test(Development):

    BROKER_URL = 'you-forgot-to-mock-celery-in-your-tests://'

+    for queue in QueueNames.all_queues():
+        Config.CELERY_QUEUES.append(
+            Queue(queue, Exchange('default'), routing_key=queue)
+        )
+
    API_RATE_LIMIT_ENABLED = True
    API_HOST_NAME = "http://localhost:6011"

--- a/app/dao/notifications_dao.py
+++ b/app/dao/notifications_dao.py
@@ -5,7 +5,7 @@ from datetime import (
    timedelta,
 )

-from botocore.exceptions import ClientError as BotoClientError
+from boto.exception import BotoClientError
 from flask import current_app

 from notifications_utils.recipients import (
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -22,7 +22,6 @@ RUN \
 		libffi-dev \
 		python-dev \
 		jq \
-		libcurl4-openssl-dev \
 	&& echo "Clean up" \
 	&& rm -rf /var/lib/apt/lists/* /tmp/*

--- a/requirements-app.txt
+++ b/requirements-app.txt
@@ -2,7 +2,7 @@
 # with package version changes made in requirements-app.txt

 cffi==1.11.5
-celery[sqs]==4.2.1
+celery==3.1.26.post2 # pyup: <4
 docopt==0.6.2
 Flask-Bcrypt==0.7.1
 flask-marshmallow==0.9.0
@@ -29,4 +29,4 @@ botocore<1.11.0

 git+https://github.com/alphagov/notifications-utils.git@30.5.3#egg=notifications-utils==30.5.3

-# if you want to update pycurl please do so in makefile
+git+https://github.com/alphagov/boto.git@2.43.0-patch3#egg=boto==2.43.0-patch3
--- a/requirements.txt
+++ b/requirements.txt
@@ -4,7 +4,7 @@
 # with package version changes made in requirements-app.txt

 cffi==1.11.5
-celery[sqs]==4.2.1
+celery==3.1.26.post2 # pyup: <4
 docopt==0.6.2
 Flask-Bcrypt==0.7.1
 flask-marshmallow==0.9.0
@@ -31,15 +31,16 @@ botocore<1.11.0

 git+https://github.com/alphagov/notifications-utils.git@30.5.3#egg=notifications-utils==30.5.3

-# if you want to update pycurl please do so in makefile
+git+https://github.com/alphagov/boto.git@2.43.0-patch3#egg=boto==2.43.0-patch3

 ## The following requirements were added by pip freeze:
 alembic==1.0.0
-amqp==2.3.2
+amqp==1.4.9
+anyjson==0.3.3
 bcrypt==3.1.4
-billiard==3.5.0.4
+billiard==3.3.0.23
 bleach==2.1.3
-boto3==1.9.17
+boto3==1.6.16
 certifi==2018.8.24
 chardet==3.0.4
 Click==7.0
@@ -53,7 +54,7 @@ idna==2.7
 itsdangerous==0.24
 Jinja2==2.10
 jmespath==0.9.3
-kombu==4.2.1
+kombu==3.0.37
 Mako==1.0.7
 MarkupSafe==1.0
 mistune==0.8.3
@@ -62,7 +63,6 @@ orderedset==2.0.1
 phonenumbers==8.9.4
 pyasn1==0.4.4
 pycparser==2.19
-pycurl==7.43.0.2
 PyPDF2==1.26.0
 python-dateutil==2.7.3
 python-editor==1.0.3
@@ -77,6 +77,5 @@ six==1.11.0
 smartypants==2.0.1
 statsd==3.2.2
 urllib3==1.23
-vine==1.1.4
 webencodings==0.5.1
 Werkzeug==0.14.1
--- a/scripts/bootstrap.sh
+++ b/scripts/bootstrap.sh
@@ -20,7 +20,16 @@ function display_result {
  fi
 }

-make build
+if [ ! $VIRTUAL_ENV ]; then
+  virtualenv -p python3 ./venv
+  . ./venv/bin/activate
+fi
+
+# we need the version file to exist otherwise the app will blow up
+make generate-version-file
+
+# Install Python development dependencies
+pip3 install -r requirements_for_test.txt

 # Create Postgres databases
 createdb notification_api