darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2026-02-01 15:46:07 -05:00
Code Issues Packages Projects Releases Wiki Activity

Revert "Celery 4"

Browse Source
This commit is contained in:
Leo Hemsted
2018-10-09 13:27:49 +01:00
committed by GitHub
parent 72358899ad
commit 2ed50e760f
7 changed files with 42 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
Makefile
View File

@@ -2,6 +2,7 @@
SHELL := /bin/bash SHELL := /bin/bash
DATE = $(shell date +%Y-%m-%d:%H:%M:%S) DATE = $(shell date +%Y-%m-%d:%H:%M:%S)
PIP_ACCEL_CACHE ?= ${CURDIR}/cache/pip-accel
APP_VERSION_FILE = app/version.py APP_VERSION_FILE = app/version.py
GIT_BRANCH ?= $(shell git symbolic-ref --short HEAD 2> /dev/null || echo "detached") GIT_BRANCH ?= $(shell git symbolic-ref --short HEAD 2> /dev/null || echo "detached")
@@ -33,19 +34,11 @@ help:
@cat $(MAKEFILE_LIST) | grep -E '^[a-zA-Z_-]+:.*?## .*$$' | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' @cat $(MAKEFILE_LIST) | grep -E '^[a-zA-Z_-]+:.*?## .*$$' | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
.PHONY: venv .PHONY: venv
# if there's a virtualenv already active, we don't want to do anything
ifeq ($(VIRTUAL_ENV),)
venv: venv/bin/activate ## Create virtualenv if it does not exist venv: venv/bin/activate ## Create virtualenv if it does not exist
venv/bin/activate: venv/bin/activate:
test -d venv || virtualenv venv -p python3 test -d venv || virtualenv venv -p python3
. venv/bin/activate && pip install pip-accel
VENV_BIN_DIR = $(shell . venv/bin/activate && echo $$VIRTUAL_ENV/bin)
else
venv:
VENV_BIN_DIR = $(VIRTUAL_ENV)/bin
endif
.PHONY: check-env-vars .PHONY: check-env-vars
check-env-vars: ## Check mandatory environment variables check-env-vars: ## Check mandatory environment variables
@@ -72,9 +65,9 @@ production: ## Set environment to production
@true @true
.PHONY: dependencies .PHONY: dependencies
dependencies: venv dependencies: venv ## Install build dependencies
$(call install-pycurl, $(VENV_BIN_DIR)/pip) mkdir -p ${PIP_ACCEL_CACHE}
$(VENV_BIN_DIR)/pip install -r requirements_for_test.txt . venv/bin/activate && PIP_ACCEL_CACHE=${PIP_ACCEL_CACHE} pip-accel install -r requirements_for_test.txt
.PHONY: generate-version-file .PHONY: generate-version-file
generate-version-file: ## Generates the app version file generate-version-file: ## Generates the app version file
@@ -82,7 +75,7 @@ generate-version-file: ## Generates the app version file
.PHONY: build .PHONY: build
build: dependencies generate-version-file ## Build project build: dependencies generate-version-file ## Build project
. venv/bin/activate && PIP_ACCEL_CACHE=${PIP_ACCEL_CACHE} pip-accel install -r requirements.txt
.PHONY: build-paas-artifact .PHONY: build-paas-artifact
build-paas-artifact: ## Build the deploy artifact for PaaS build-paas-artifact: ## Build the deploy artifact for PaaS
@@ -104,7 +97,6 @@ test: venv generate-version-file ## Run tests
freeze-requirements: freeze-requirements:
rm -rf venv-freeze rm -rf venv-freeze
virtualenv -p python3 venv-freeze virtualenv -p python3 venv-freeze
$(call install-pycurl, $$(pwd)/venv-freeze/bin/pip)
$$(pwd)/venv-freeze/bin/pip install -r requirements-app.txt $$(pwd)/venv-freeze/bin/pip install -r requirements-app.txt
echo '# pyup: ignore file' > requirements.txt echo '# pyup: ignore file' > requirements.txt
echo '# This file is autogenerated. Do not edit it manually.' >> requirements.txt echo '# This file is autogenerated. Do not edit it manually.' >> requirements.txt
@@ -113,15 +105,6 @@ freeze-requirements:
$$(pwd)/venv-freeze/bin/pip freeze -r <(sed '/^--/d' requirements-app.txt) | sed -n '/The following requirements were added by pip freeze/,$$p' >> requirements.txt $$(pwd)/venv-freeze/bin/pip freeze -r <(sed '/^--/d' requirements-app.txt) | sed -n '/The following requirements were added by pip freeze/,$$p' >> requirements.txt
rm -rf venv-freeze rm -rf venv-freeze
define install-pycurl
# install pycurl separately to avoid flags disabling wheels for other packages
PYCURL_SSL_LIBRARY=openssl ${1} install pycurl==7.43.0.2 --global-option="build_ext" --global-option="-I/usr/local/opt/openssl/include"
endef
.PHONY: install-pycurl
install-pycurl:
$(call install-pycurl, pip)
.PHONY: test-requirements .PHONY: test-requirements
test-requirements: test-requirements:
@diff requirements-app.txt requirements.txt | grep '<' \ @diff requirements-app.txt requirements.txt | grep '<' \
@@ -131,10 +114,11 @@ test-requirements:
.PHONY: coverage .PHONY: coverage
coverage: venv ## Create coverage report coverage: venv ## Create coverage report
coveralls . venv/bin/activate && coveralls
.PHONY: prepare-docker-build-image .PHONY: prepare-docker-build-image
prepare-docker-build-image: ## Prepare the Docker builder image prepare-docker-build-image: ## Prepare the Docker builder image
mkdir -p ${PIP_ACCEL_CACHE}
make -C docker build make -C docker build
.PHONY: build-with-docker .PHONY: build-with-docker
@@ -142,6 +126,7 @@ build-with-docker: prepare-docker-build-image ## Build inside a Docker container
@docker run -i${DOCKER_TTY} --rm \ @docker run -i${DOCKER_TTY} --rm \
--name "${DOCKER_CONTAINER_PREFIX}-build" \ --name "${DOCKER_CONTAINER_PREFIX}-build" \
-v "`pwd`:/var/project" \ -v "`pwd`:/var/project" \
-v "${PIP_ACCEL_CACHE}:/var/project/cache/pip-accel" \
-e UID=$(shell id -u) \ -e UID=$(shell id -u) \
-e GID=$(shell id -g) \ -e GID=$(shell id -g) \
-e GIT_COMMIT=${GIT_COMMIT} \ -e GIT_COMMIT=${GIT_COMMIT} \

18
app/config.py
View File

@@ -150,9 +150,9 @@ class Config(object):
BROKER_URL = 'sqs://' BROKER_URL = 'sqs://'
BROKER_TRANSPORT_OPTIONS = { BROKER_TRANSPORT_OPTIONS = {
'region': AWS_REGION, 'region': AWS_REGION,
'polling_interval': 1, # 1 second
'visibility_timeout': 310, 'visibility_timeout': 310,
'queue_name_prefix': NOTIFICATION_QUEUE_PREFIX, 'queue_name_prefix': NOTIFICATION_QUEUE_PREFIX
'wait_time_seconds': 20 # enable long polling, with a wait time of 20 seconds
} }
CELERY_ENABLE_UTC = True CELERY_ENABLE_UTC = True
CELERY_TIMEZONE = 'Europe/London' CELERY_TIMEZONE = 'Europe/London'
@@ -270,9 +270,7 @@ class Config(object):
'options': {'queue': QueueNames.PERIODIC} 'options': {'queue': QueueNames.PERIODIC}
} }
} }
CELERY_QUEUES = []
# this is overriden by the -Q command, but locally, we should read from all queues
CELERY_QUEUES = [Queue(queue, Exchange('default'), routing_key=queue) for queue in QueueNames.all_queues()]
NOTIFICATIONS_ALERT = 5 # five mins NOTIFICATIONS_ALERT = 5 # five mins
FROM_NUMBER = 'development' FROM_NUMBER = 'development'
@@ -360,6 +358,11 @@ class Development(Config):
STATSD_PORT = 1000 STATSD_PORT = 1000
STATSD_PREFIX = "stats-prefix" STATSD_PREFIX = "stats-prefix"
for queue in QueueNames.all_queues():
Config.CELERY_QUEUES.append(
Queue(queue, Exchange('default'), routing_key=queue)
)
API_HOST_NAME = "http://localhost:6011" API_HOST_NAME = "http://localhost:6011"
API_RATE_LIMIT_ENABLED = True API_RATE_LIMIT_ENABLED = True
@@ -382,6 +385,11 @@ class Test(Development):
BROKER_URL = 'you-forgot-to-mock-celery-in-your-tests://' BROKER_URL = 'you-forgot-to-mock-celery-in-your-tests://'
for queue in QueueNames.all_queues():
Config.CELERY_QUEUES.append(
Queue(queue, Exchange('default'), routing_key=queue)
)
API_RATE_LIMIT_ENABLED = True API_RATE_LIMIT_ENABLED = True
API_HOST_NAME = "http://localhost:6011" API_HOST_NAME = "http://localhost:6011"

2
app/dao/notifications_dao.py
View File

@@ -5,7 +5,7 @@ from datetime import (
timedelta, timedelta,
) )
from botocore.exceptions import ClientError as BotoClientError from boto.exception import BotoClientError
from flask import current_app from flask import current_app
from notifications_utils.recipients import ( from notifications_utils.recipients import (

1
docker/Dockerfile
View File

@@ -22,7 +22,6 @@ RUN \
libffi-dev \ libffi-dev \
python-dev \ python-dev \
jq \ jq \
libcurl4-openssl-dev \
&& echo "Clean up" \ && echo "Clean up" \
&& rm -rf /var/lib/apt/lists/* /tmp/* && rm -rf /var/lib/apt/lists/* /tmp/*

4
requirements-app.txt
View File

@@ -2,7 +2,7 @@
# with package version changes made in requirements-app.txt # with package version changes made in requirements-app.txt
cffi==1.11.5 cffi==1.11.5
celery[sqs]==4.2.1 celery==3.1.26.post2 # pyup: <4
docopt==0.6.2 docopt==0.6.2
Flask-Bcrypt==0.7.1 Flask-Bcrypt==0.7.1
flask-marshmallow==0.9.0 flask-marshmallow==0.9.0
@@ -29,4 +29,4 @@ botocore<1.11.0
git+https://github.com/alphagov/notifications-utils.git@30.5.3#egg=notifications-utils==30.5.3 git+https://github.com/alphagov/notifications-utils.git@30.5.3#egg=notifications-utils==30.5.3
# if you want to update pycurl please do so in makefile git+https://github.com/alphagov/boto.git@2.43.0-patch3#egg=boto==2.43.0-patch3

15
requirements.txt
View File

@@ -4,7 +4,7 @@
# with package version changes made in requirements-app.txt # with package version changes made in requirements-app.txt
cffi==1.11.5 cffi==1.11.5
celery[sqs]==4.2.1 celery==3.1.26.post2 # pyup: <4
docopt==0.6.2 docopt==0.6.2
Flask-Bcrypt==0.7.1 Flask-Bcrypt==0.7.1
flask-marshmallow==0.9.0 flask-marshmallow==0.9.0
@@ -31,15 +31,16 @@ botocore<1.11.0
git+https://github.com/alphagov/notifications-utils.git@30.5.3#egg=notifications-utils==30.5.3 git+https://github.com/alphagov/notifications-utils.git@30.5.3#egg=notifications-utils==30.5.3
# if you want to update pycurl please do so in makefile git+https://github.com/alphagov/boto.git@2.43.0-patch3#egg=boto==2.43.0-patch3
## The following requirements were added by pip freeze: ## The following requirements were added by pip freeze:
alembic==1.0.0 alembic==1.0.0
amqp==2.3.2 amqp==1.4.9
anyjson==0.3.3
bcrypt==3.1.4 bcrypt==3.1.4
billiard==3.5.0.4 billiard==3.3.0.23
bleach==2.1.3 bleach==2.1.3
boto3==1.9.17 boto3==1.6.16
certifi==2018.8.24 certifi==2018.8.24
chardet==3.0.4 chardet==3.0.4
Click==7.0 Click==7.0
@@ -53,7 +54,7 @@ idna==2.7
itsdangerous==0.24 itsdangerous==0.24
Jinja2==2.10 Jinja2==2.10
jmespath==0.9.3 jmespath==0.9.3
kombu==4.2.1 kombu==3.0.37
Mako==1.0.7 Mako==1.0.7
MarkupSafe==1.0 MarkupSafe==1.0
mistune==0.8.3 mistune==0.8.3
@@ -62,7 +63,6 @@ orderedset==2.0.1
phonenumbers==8.9.4 phonenumbers==8.9.4
pyasn1==0.4.4 pyasn1==0.4.4
pycparser==2.19 pycparser==2.19
pycurl==7.43.0.2
PyPDF2==1.26.0 PyPDF2==1.26.0
python-dateutil==2.7.3 python-dateutil==2.7.3
python-editor==1.0.3 python-editor==1.0.3
@@ -77,6 +77,5 @@ six==1.11.0
smartypants==2.0.1 smartypants==2.0.1
statsd==3.2.2 statsd==3.2.2
urllib3==1.23 urllib3==1.23
vine==1.1.4
webencodings==0.5.1 webencodings==0.5.1
Werkzeug==0.14.1 Werkzeug==0.14.1

11
scripts/bootstrap.sh
View File

@@ -20,7 +20,16 @@ function display_result {
fi fi
} }
make build if [ ! $VIRTUAL_ENV ]; then
virtualenv -p python3 ./venv
. ./venv/bin/activate
fi
# we need the version file to exist otherwise the app will blow up
make generate-version-file
# Install Python development dependencies
pip3 install -r requirements_for_test.txt
# Create Postgres databases # Create Postgres databases
createdb notification_api createdb notification_api