diff --git a/Makefile b/Makefile index 3612540bc..e94889cea 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,6 @@ SHELL := /bin/bash DATE = $(shell date +%Y-%m-%d:%H:%M:%S) -PIP_ACCEL_CACHE ?= ${CURDIR}/cache/pip-accel APP_VERSION_FILE = app/version.py GIT_BRANCH ?= $(shell git symbolic-ref --short HEAD 2> /dev/null || echo "detached") @@ -34,11 +33,19 @@ help: @cat $(MAKEFILE_LIST) | grep -E '^[a-zA-Z_-]+:.*?## .*$$' | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' .PHONY: venv +# if there's a virtualenv already active, we don't want to do anything +ifeq ($(VIRTUAL_ENV),) venv: venv/bin/activate ## Create virtualenv if it does not exist venv/bin/activate: test -d venv || virtualenv venv -p python3 - . venv/bin/activate && pip install pip-accel + +VENV_BIN_DIR = $(shell . venv/bin/activate && echo $$VIRTUAL_ENV/bin) +else +venv: + +VENV_BIN_DIR = $(VIRTUAL_ENV)/bin +endif .PHONY: check-env-vars check-env-vars: ## Check mandatory environment variables @@ -65,9 +72,9 @@ production: ## Set environment to production @true .PHONY: dependencies -dependencies: venv ## Install build dependencies - mkdir -p ${PIP_ACCEL_CACHE} - . venv/bin/activate && PIP_ACCEL_CACHE=${PIP_ACCEL_CACHE} pip-accel install -r requirements_for_test.txt +dependencies: venv + $(call install-pycurl, $(VENV_BIN_DIR)/pip) + $(VENV_BIN_DIR)/pip install -r requirements_for_test.txt .PHONY: generate-version-file generate-version-file: ## Generates the app version file @@ -75,7 +82,7 @@ generate-version-file: ## Generates the app version file .PHONY: build build: dependencies generate-version-file ## Build project - . venv/bin/activate && PIP_ACCEL_CACHE=${PIP_ACCEL_CACHE} pip-accel install -r requirements.txt + .PHONY: build-paas-artifact build-paas-artifact: ## Build the deploy artifact for PaaS @@ -97,6 +104,7 @@ test: venv generate-version-file ## Run tests freeze-requirements: rm -rf venv-freeze virtualenv -p python3 venv-freeze + $(call install-pycurl, $$(pwd)/venv-freeze/bin/pip) $$(pwd)/venv-freeze/bin/pip install -r requirements-app.txt echo '# pyup: ignore file' > requirements.txt echo '# This file is autogenerated. Do not edit it manually.' >> requirements.txt @@ -105,6 +113,15 @@ freeze-requirements: $$(pwd)/venv-freeze/bin/pip freeze -r <(sed '/^--/d' requirements-app.txt) | sed -n '/The following requirements were added by pip freeze/,$$p' >> requirements.txt rm -rf venv-freeze +define install-pycurl + # install pycurl separately to avoid flags disabling wheels for other packages + PYCURL_SSL_LIBRARY=openssl ${1} install pycurl==7.43.0.2 --global-option="build_ext" --global-option="-I/usr/local/opt/openssl/include" +endef + +.PHONY: install-pycurl +install-pycurl: + $(call install-pycurl, pip) + .PHONY: test-requirements test-requirements: @diff requirements-app.txt requirements.txt | grep '<' \ @@ -114,11 +131,10 @@ test-requirements: .PHONY: coverage coverage: venv ## Create coverage report - . venv/bin/activate && coveralls + coveralls .PHONY: prepare-docker-build-image prepare-docker-build-image: ## Prepare the Docker builder image - mkdir -p ${PIP_ACCEL_CACHE} make -C docker build .PHONY: build-with-docker @@ -126,7 +142,6 @@ build-with-docker: prepare-docker-build-image ## Build inside a Docker container @docker run -i${DOCKER_TTY} --rm \ --name "${DOCKER_CONTAINER_PREFIX}-build" \ -v "`pwd`:/var/project" \ - -v "${PIP_ACCEL_CACHE}:/var/project/cache/pip-accel" \ -e UID=$(shell id -u) \ -e GID=$(shell id -g) \ -e GIT_COMMIT=${GIT_COMMIT} \ diff --git a/app/config.py b/app/config.py index ae7788d34..c37dd5d00 100644 --- a/app/config.py +++ b/app/config.py @@ -150,9 +150,9 @@ class Config(object): BROKER_URL = 'sqs://' BROKER_TRANSPORT_OPTIONS = { 'region': AWS_REGION, - 'polling_interval': 1, # 1 second 'visibility_timeout': 310, - 'queue_name_prefix': NOTIFICATION_QUEUE_PREFIX + 'queue_name_prefix': NOTIFICATION_QUEUE_PREFIX, + 'wait_time_seconds': 20 # enable long polling, with a wait time of 20 seconds } CELERY_ENABLE_UTC = True CELERY_TIMEZONE = 'Europe/London' @@ -270,7 +270,9 @@ class Config(object): 'options': {'queue': QueueNames.PERIODIC} } } - CELERY_QUEUES = [] + + # this is overriden by the -Q command, but locally, we should read from all queues + CELERY_QUEUES = [Queue(queue, Exchange('default'), routing_key=queue) for queue in QueueNames.all_queues()] NOTIFICATIONS_ALERT = 5 # five mins FROM_NUMBER = 'development' @@ -358,11 +360,6 @@ class Development(Config): STATSD_PORT = 1000 STATSD_PREFIX = "stats-prefix" - for queue in QueueNames.all_queues(): - Config.CELERY_QUEUES.append( - Queue(queue, Exchange('default'), routing_key=queue) - ) - API_HOST_NAME = "http://localhost:6011" API_RATE_LIMIT_ENABLED = True @@ -385,11 +382,6 @@ class Test(Development): BROKER_URL = 'you-forgot-to-mock-celery-in-your-tests://' - for queue in QueueNames.all_queues(): - Config.CELERY_QUEUES.append( - Queue(queue, Exchange('default'), routing_key=queue) - ) - API_RATE_LIMIT_ENABLED = True API_HOST_NAME = "http://localhost:6011" diff --git a/app/dao/notifications_dao.py b/app/dao/notifications_dao.py index 95896e4da..6cfca54a8 100644 --- a/app/dao/notifications_dao.py +++ b/app/dao/notifications_dao.py @@ -5,7 +5,7 @@ from datetime import ( timedelta, ) -from boto.exception import BotoClientError +from botocore.exceptions import ClientError as BotoClientError from flask import current_app from notifications_utils.recipients import ( diff --git a/docker/Dockerfile b/docker/Dockerfile index 1060824cf..db1bc6202 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,6 +22,7 @@ RUN \ libffi-dev \ python-dev \ jq \ + libcurl4-openssl-dev \ && echo "Clean up" \ && rm -rf /var/lib/apt/lists/* /tmp/* diff --git a/requirements-app.txt b/requirements-app.txt index 50b70fb08..d656a6e2a 100644 --- a/requirements-app.txt +++ b/requirements-app.txt @@ -2,7 +2,7 @@ # with package version changes made in requirements-app.txt cffi==1.11.5 -celery==3.1.26.post2 # pyup: <4 +celery[sqs]==4.2.1 docopt==0.6.2 Flask-Bcrypt==0.7.1 flask-marshmallow==0.9.0 @@ -29,4 +29,4 @@ botocore<1.11.0 git+https://github.com/alphagov/notifications-utils.git@30.5.2#egg=notifications-utils==30.5.2 -git+https://github.com/alphagov/boto.git@2.43.0-patch3#egg=boto==2.43.0-patch3 +# if you want to update pycurl please do so in makefile diff --git a/requirements.txt b/requirements.txt index 80b9ec243..2952cac2a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ # with package version changes made in requirements-app.txt cffi==1.11.5 -celery==3.1.26.post2 # pyup: <4 +celery[sqs]==4.2.1 docopt==0.6.2 Flask-Bcrypt==0.7.1 flask-marshmallow==0.9.0 @@ -31,16 +31,15 @@ botocore<1.11.0 git+https://github.com/alphagov/notifications-utils.git@30.5.2#egg=notifications-utils==30.5.2 -git+https://github.com/alphagov/boto.git@2.43.0-patch3#egg=boto==2.43.0-patch3 +# if you want to update pycurl please do so in makefile ## The following requirements were added by pip freeze: alembic==1.0.0 -amqp==1.4.9 -anyjson==0.3.3 +amqp==2.3.2 bcrypt==3.1.4 -billiard==3.3.0.23 +billiard==3.5.0.4 bleach==2.1.3 -boto3==1.6.16 +boto3==1.9.17 certifi==2018.8.24 chardet==3.0.4 Click==7.0 @@ -54,7 +53,7 @@ idna==2.7 itsdangerous==0.24 Jinja2==2.10 jmespath==0.9.3 -kombu==3.0.37 +kombu==4.2.1 Mako==1.0.7 MarkupSafe==1.0 mistune==0.8.3 @@ -63,6 +62,7 @@ orderedset==2.0.1 phonenumbers==8.9.4 pyasn1==0.4.4 pycparser==2.19 +pycurl==7.43.0.2 PyPDF2==1.26.0 python-dateutil==2.7.3 python-editor==1.0.3 @@ -77,5 +77,6 @@ six==1.11.0 smartypants==2.0.1 statsd==3.2.2 urllib3==1.23 +vine==1.1.4 webencodings==0.5.1 Werkzeug==0.14.1 diff --git a/scripts/bootstrap.sh b/scripts/bootstrap.sh index c46d03aa6..b92832233 100755 --- a/scripts/bootstrap.sh +++ b/scripts/bootstrap.sh @@ -20,16 +20,7 @@ function display_result { fi } -if [ ! $VIRTUAL_ENV ]; then - virtualenv -p python3 ./venv - . ./venv/bin/activate -fi - -# we need the version file to exist otherwise the app will blow up -make generate-version-file - -# Install Python development dependencies -pip3 install -r requirements_for_test.txt +make build # Create Postgres databases createdb notification_api