Merge pull request #1532 from alphagov/clean-makefile

Clean makefile

Makefile

@@ -41,32 +41,25 @@ venv/bin/activate:
 .PHONY: check-env-vars
 check-env-vars: ## Check mandatory environment variables
 	$(if ${DEPLOY_ENV},,$(error Must specify DEPLOY_ENV))
-	$(if ${DNS_NAME},,$(error Must specify DNS_NAME))
-	$(if ${AWS_ACCESS_KEY_ID},,$(error Must specify AWS_ACCESS_KEY_ID))
-	$(if ${AWS_SECRET_ACCESS_KEY},,$(error Must specify AWS_SECRET_ACCESS_KEY))
 
 .PHONY: sandbox
 sandbox: ## Set environment to sandbox
 	$(eval export DEPLOY_ENV=sandbox)
-	$(eval export DNS_NAME="cloudapps.digital")
 	@true
 
 .PHONY: preview
 preview: ## Set environment to preview
 	$(eval export DEPLOY_ENV=preview)
-	$(eval export DNS_NAME="notify.works")
 	@true
 
 .PHONY: staging
 staging: ## Set environment to staging
 	$(eval export DEPLOY_ENV=staging)
-	$(eval export DNS_NAME="staging-notify.works")
 	@true
 
 .PHONY: production
 production: ## Set environment to production
 	$(eval export DEPLOY_ENV=production)
-	$(eval export DNS_NAME="notifications.service.gov.uk")
 	@true
 
 .PHONY: dependencies
@@ -82,31 +75,12 @@ generate-version-file: ## Generates the app version file
 build: dependencies generate-version-file ## Build project
 	. venv/bin/activate && PIP_ACCEL_CACHE=${PIP_ACCEL_CACHE} pip-accel install -r requirements.txt
 
-.PHONY: cf-build
+.PHONY: build-paas-artifact
-cf-build: dependencies generate-version-file ## Build project for PAAS
+build-paas-artifact:  ## Build the deploy artifact for PaaS
-
-.PHONY: build-codedeploy-artifact
-build-codedeploy-artifact: ## Build the deploy artifact for CodeDeploy
 	rm -rf target
 	mkdir -p target
 	zip -y -q -r -x@deploy-exclude.lst target/notifications-api.zip ./
 
-	rm -rf build/db-migration-codedeploy
-	mkdir -p build/db-migration-codedeploy
-	unzip target/notifications-api.zip -d build/db-migration-codedeploy
-	cd build/db-migration-codedeploy && \
-		mv -f appspec-db-migration.yml appspec.yml && \
-		zip -y -q -r -x@deploy-exclude.lst ../../target/notifications-api-db-migration.zip ./
-
-.PHONY: upload-codedeploy-artifact ## Upload the deploy artifact for CodeDeploy
-upload-codedeploy-artifact: check-env-vars
-	$(if ${DEPLOY_BUILD_NUMBER},,$(error Must specify DEPLOY_BUILD_NUMBER))
-	aws s3 cp --region eu-west-1 --sse AES256 target/notifications-api.zip s3://${DNS_NAME}-codedeploy/notifications-api-${DEPLOY_BUILD_NUMBER}.zip
-	aws s3 cp --region eu-west-1 --sse AES256 target/notifications-api-db-migration.zip s3://${DNS_NAME}-codedeploy/notifications-api-db-migration-${DEPLOY_BUILD_NUMBER}.zip
-
-.PHONY: build-paas-artifact
-build-paas-artifact: build-codedeploy-artifact ## Build the deploy artifact for PaaS
-
 .PHONY: upload-paas-artifact ## Upload the deploy artifact for PaaS
 upload-paas-artifact:
 	$(if ${DEPLOY_BUILD_NUMBER},,$(error Must specify DEPLOY_BUILD_NUMBER))
@@ -117,42 +91,6 @@ upload-paas-artifact:
 test: venv generate-version-file ## Run tests
 	./scripts/run_tests.sh
 
-.PHONY: deploy-api
-deploy-api: check-env-vars ## Trigger CodeDeploy for the api
-	aws deploy create-deployment --application-name notify-api --deployment-config-name CodeDeployDefault.OneAtATime --deployment-group-name notify-api --s3-location bucket=${DNS_NAME}-codedeploy,key=notifications-api-${DEPLOY_BUILD_NUMBER}.zip,bundleType=zip --region eu-west-1
-
-.PHONY: deploy-api
-deploy-api-db-migration: check-env-vars ## Trigger CodeDeploy for the api db migration
-	aws deploy create-deployment --application-name notify-api-db-migration --deployment-config-name CodeDeployDefault.OneAtATime --deployment-group-name notify-api-db-migration --s3-location bucket=${DNS_NAME}-codedeploy,key=notifications-api-db-migration-${DEPLOY_BUILD_NUMBER}.zip,bundleType=zip --region eu-west-1
-
-.PHONY: deploy-admin-api
-deploy-admin-api: check-env-vars ## Trigger CodeDeploy for the admin api
-	aws deploy create-deployment --application-name notify-admin-api --deployment-config-name CodeDeployDefault.OneAtATime --deployment-group-name notify-admin-api --s3-location bucket=${DNS_NAME}-codedeploy,key=notifications-api-${DEPLOY_BUILD_NUMBER}.zip,bundleType=zip --region eu-west-1
-
-.PHONY: deploy-delivery
-deploy-delivery: check-env-vars ## Trigger CodeDeploy for the delivery app
-	aws deploy create-deployment --application-name notify-delivery --deployment-config-name CodeDeployDefault.OneAtATime --deployment-group-name notify-delivery --s3-location bucket=${DNS_NAME}-codedeploy,key=notifications-api-${DEPLOY_BUILD_NUMBER}.zip,bundleType=zip --region eu-west-1
-
-.PHONY: check-aws-vars
-check-aws-vars: ## Check if AWS access keys are set
-	$(if ${AWS_ACCESS_KEY_ID},,$(error Must specify AWS_ACCESS_KEY_ID))
-	$(if ${AWS_SECRET_ACCESS_KEY},,$(error Must specify AWS_SECRET_ACCESS_KEY))
-
-.PHONY: deploy-suspend-autoscaling-processes
-deploy-suspend-autoscaling-processes: check-aws-vars ## Suspend launch and terminate processes for the auto-scaling group
-	$(if ${CODEDEPLOY_APP_NAME},,$(error Must specify CODEDEPLOY_APP_NAME))
-	aws autoscaling suspend-processes --region eu-west-1 --auto-scaling-group-name ${CODEDEPLOY_APP_NAME} --scaling-processes "Launch" "Terminate"
-
-.PHONY: deploy-resume-autoscaling-processes
-deploy-resume-autoscaling-processes: check-aws-vars ## Resume launch and terminate processes for the auto-scaling group
-	$(if ${CODEDEPLOY_APP_NAME},,$(error Must specify CODEDEPLOY_APP_NAME))
-	aws autoscaling resume-processes --region eu-west-1 --auto-scaling-group-name ${CODEDEPLOY_APP_NAME} --scaling-processes "Launch" "Terminate"
-
-.PHONY: deploy-check-autoscaling-processes
-deploy-check-autoscaling-processes: check-aws-vars ## Returns with the number of instances with active autoscaling events
-	$(if ${CODEDEPLOY_APP_NAME},,$(error Must specify CODEDEPLOY_APP_NAME))
-	@aws autoscaling describe-auto-scaling-groups --region eu-west-1 --auto-scaling-group-names ${CODEDEPLOY_APP_NAME} | jq '.AutoScalingGroups[0].Instances|map(select(.LifecycleState != "InService"))|length'
-
 .PHONY: coverage
 coverage: venv ## Create coverage report
 	. venv/bin/activate && coveralls
@@ -181,25 +119,6 @@ build-with-docker: prepare-docker-build-image ## Build inside a Docker container
 		${DOCKER_BUILDER_IMAGE_NAME} \
 		gosu hostuser make build
 
-.PHONY: cf-build-with-docker
-cf-build-with-docker: prepare-docker-build-image ## Build inside a Docker container
-	@docker run -i${DOCKER_TTY} --rm \
-		--name "${DOCKER_CONTAINER_PREFIX}-build" \
-		-v "`pwd`:/var/project" \
-		-v "${PIP_ACCEL_CACHE}:/var/project/cache/pip-accel" \
-		-e UID=$(shell id -u) \
-		-e GID=$(shell id -g) \
-		-e GIT_COMMIT=${GIT_COMMIT} \
-		-e BUILD_NUMBER=${BUILD_NUMBER} \
-		-e BUILD_URL=${BUILD_URL} \
-		-e http_proxy="${HTTP_PROXY}" \
-		-e HTTP_PROXY="${HTTP_PROXY}" \
-		-e https_proxy="${HTTPS_PROXY}" \
-		-e HTTPS_PROXY="${HTTPS_PROXY}" \
-		-e NO_PROXY="${NO_PROXY}" \
-		${DOCKER_BUILDER_IMAGE_NAME} \
-		gosu hostuser make cf-build
-
 .PHONY: test-with-docker
 test-with-docker: prepare-docker-build-image create-docker-test-db ## Run tests inside a Docker container
 	@docker run -i${DOCKER_TTY} --rm \
@@ -220,7 +139,7 @@ test-with-docker: prepare-docker-build-image create-docker-test-db ## Run tests
 		${DOCKER_BUILDER_IMAGE_NAME} \
 		gosu hostuser make test
 
-.PHONY: test-with-docker
+.PHONY: create-docker-test-db
 create-docker-test-db: ## Start the test database in a Docker container
 	docker rm -f ${DOCKER_CONTAINER_PREFIX}-db 2> /dev/null || true
 	@docker run -d \

appspec-db-migration.yml

@@ -1,18 +0,0 @@
----
-os: linux
-version: 0.0
-files:
-  - destination: /home/notify-app/notifications-api
-    source: /
-hooks:
-  AfterInstall:
-    - location: scripts/aws_install_dependencies.sh
-      runas: root
-      timeout: 1000
-    - location: scripts/aws_change_ownership.sh
-      runas: root
-      timeout: 300
-  ApplicationStart:
-    - location: scripts/aws_run_db_migrations.sh
-      runas: root
-      timeout: 300

appspec.yml

@@ -1,32 +0,0 @@
----
-os: linux
-version: 0.0
-files:
-  - destination: /home/notify-app/notifications-api
-    source: /
-hooks:
-  BeforeInstall:
-    - location: scripts/aws_clear_instance.sh
-      runas: root
-      timeout: 1000
-  AfterInstall:
-    - location: scripts/aws_install_dependencies.sh
-      runas: root
-      timeout: 1000
-    - location: scripts/aws_change_ownership.sh
-      runas: root
-      timeout: 300
-  ApplicationStart:
-    - location: scripts/aws_start_app.sh
-      runas: root
-      timeout: 300
-    - location: scripts/register_with_elb.sh
-      runas: ubuntu
-      timeout: 300
-  ApplicationStop:
-    - location: scripts/deregister_from_elb.sh
-      runas: ubuntu
-      timeout: 300
-    - location: scripts/aws_stop_app.sh
-      runas: root
-      timeout: 300

scripts/aws_change_ownership.sh

@@ -1,8 +0,0 @@
-#!/bin/bash
-
-set -eo pipefail
-
-echo "Chown application to be owned by notify-app"
-
-cd /home/notify-app/;
-chown -R notify-app:govuk-notify-applications notifications-api

scripts/aws_clear_instance.sh

@@ -1,10 +0,0 @@
-#!/bin/bash
-
-echo "Removing application and dependencies"
-
-if [ -d "/home/notify-app/notifications-api" ]; then
-    # Remove and re-create the directory
-    rm -rf /home/notify-app/notifications-api
-    mkdir -p /home/notify-app/notifications-api
-fi
-

scripts/aws_install_dependencies.sh

@@ -1,8 +0,0 @@
-#!/bin/bash
-
-set -eo pipefail
-
-echo "Install dependencies"
-
-cd /home/notify-app/notifications-api;
-pip3 install -r /home/notify-app/notifications-api/requirements.txt

scripts/aws_run_db_migrations.sh

@@ -1,8 +0,0 @@
-#!/usr/bin/env bash
-
-set -eo pipefail
-
-echo "Run database migrations"
-
-cd /home/notify-app/notifications-api;
-python3 db.py db upgrade

scripts/aws_start_app.sh

@@ -1,20 +0,0 @@
-#!/usr/bin/env bash
-
-set -eo pipefail
-
-function start
-{
-  service=$1
-  if [ -e "/etc/init/${service}.conf" ]
-  then
-    echo "Starting ${service}"
-    service ${service} start
-  fi
-}
-
-start "notifications-api"
-start "notifications-api-celery-worker"
-start "notifications-api-celery-worker-sender"
-start "notifications-api-celery-worker-research"
-start "notifications-api-celery-worker-db"
-start "notifications-api-celery-beat"

scripts/aws_stop_app.sh

@@ -1,23 +0,0 @@
-#!/usr/bin/env bash
-
-set -eo pipefail
-
-function stop
-{
-  service=$1
-  if [ -e "/etc/init/${service}.conf" ]; then
-    echo "stopping ${service}"
-    if service ${service} stop; then
-      echo "${service} stopped"
-    else
-      >&2 echo "Could not stop ${service}"
-    fi
-  fi
-}
-
-stop "notifications-api"
-stop "notifications-api-celery-beat"
-stop "notifications-api-celery-worker"
-stop "notifications-api-celery-worker-sender"
-stop "notifications-api-celery-worker-research"
-stop "notifications-api-celery-worker-db"

scripts/deregister_from_elb.sh

@@ -1,77 +0,0 @@
-#!/bin/bash
-#
-# Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License").
-# You may not use this file except in compliance with the License.
-# A copy of the License is located at
-#
-#  http://aws.amazon.com/apache2.0
-#
-# or in the "license" file accompanying this file. This file is distributed
-# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
-# express or implied. See the License for the specific language governing
-# permissions and limitations under the License.
-
-. $(dirname $0)/common_functions.sh
-
-msg "Running AWS CLI with region: $(get_instance_region)"
-
-# get this instance's ID
-INSTANCE_ID=$(get_instance_id)
-if [ $? != 0 -o -z "$INSTANCE_ID" ]; then
-    error_exit "Unable to get this instance's ID; cannot continue."
-fi
-
-# Get current time
-msg "Started $(basename $0) at $(/bin/date "+%F %T")"
-start_sec=$(/bin/date +%s.%N)
-
-msg "Getting relevant load balancer"
-INSTANCE_NAME=$(get_instance_name_from_tags $INSTANCE_ID)
-
-if [[ "$(tr [:upper:] [:lower:] <<< "${INSTANCE_NAME}")" =~ 'delivery' ]]; then
-    msg "NO ELBs for delivery"
-    exit 0
-fi
-
-get_elb_name_for_instance_name $INSTANCE_NAME
-get_elb_list $INSTANCE_ID $ELB_NAME
-
-msg "Checking that user set at least one load balancer"
-if test -z "$ELB_LIST"; then
-    error_exit "Must have at least one load balancer to deregister from"
-fi
-
-# Loop through all LBs the user set, and attempt to deregister this instance from them.
-for elb in $ELB_LIST; do
-    msg "Checking validity of load balancer named '$elb'"
-    validate_elb $INSTANCE_ID $elb
-    if [ $? != 0 ]; then
-        msg "Error validating $elb; cannot continue with this LB"
-        continue
-    fi
-
-    msg "Deregistering $INSTANCE_ID from $elb"
-    deregister_instance $INSTANCE_ID $elb
-
-    if [ $? != 0 ]; then
-        error_exit "Failed to deregister instance $INSTANCE_ID from ELB $elb"
-    fi
-done
-
-# Wait for all Deregistrations to finish
-msg "Waiting for instance to de-register from its load balancers"
-for elb in $ELB_LIST; do
-    wait_for_state "elb" $INSTANCE_ID "OutOfService" $elb
-    if [ $? != 0 ]; then
-        error_exit "Failed waiting for $INSTANCE_ID to leave $elb"
-    fi
-done
-
-msg "Finished $(basename $0) at $(/bin/date "+%F %T")"
-
-end_sec=$(/bin/date +%s.%N)
-elapsed_seconds=$(echo "$end_sec - $start_sec" | /usr/bin/bc)
-
-msg "Elapsed time: $elapsed_seconds"

scripts/register_with_elb.sh

@@ -1,78 +0,0 @@
-#!/bin/bash
-#
-# Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License").
-# You may not use this file except in compliance with the License.
-# A copy of the License is located at
-#
-#  http://aws.amazon.com/apache2.0
-#
-# or in the "license" file accompanying this file. This file is distributed
-# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
-# express or implied. See the License for the specific language governing
-# permissions and limitations under the License.
-
-. $(dirname $0)/common_functions.sh
-
-msg "Running AWS CLI with region: $(get_instance_region)"
-
-# get this instance's ID
-INSTANCE_ID=$(get_instance_id)
-if [ $? != 0 -o -z "$INSTANCE_ID" ]; then
-    error_exit "Unable to get this instance's ID; cannot continue."
-fi
-
-# Get current time
-msg "Started $(basename $0) at $(/bin/date "+%F %T")"
-start_sec=$(/bin/date +%s.%N)
-
-msg "Getting relevant load balancer"
-INSTANCE_NAME=$(get_instance_name_from_tags $INSTANCE_ID)
-
-if [[ "$(tr [:upper:] [:lower:] <<< "${INSTANCE_NAME}")" =~ 'delivery' ]]; then
-    msg "NO ELBs for delivery"
-    exit 0
-fi
-
-get_elb_name_for_instance_name $INSTANCE_NAME
-ELB_LIST=$ELB_NAME
-get_elb_list $INSTANCE_ID $ELB_NAME
-
-msg "Checking that user set at least one load balancer"
-if test -z "$ELB_LIST"; then
-    error_exit "Must have at least one load balancer to register to"
-fi
-
-# Loop through all LBs the user set, and attempt to register this instance to them.
-for elb in $ELB_LIST; do
-    msg "Checking validity of load balancer named '$elb'"
-    validate_elb $INSTANCE_ID $elb
-    if [ $? != 0 ]; then
-        msg "Error validating $elb; cannot continue with this LB"
-        continue
-    fi
-
-    msg "Registering $INSTANCE_ID to $elb"
-    register_instance $INSTANCE_ID $elb
-
-    if [ $? != 0 ]; then
-        error_exit "Failed to register instance $INSTANCE_ID from ELB $elb"
-    fi
-done
-
-# Wait for all Registrations to finish
-msg "Waiting for instance to register to its load balancers"
-for elb in $ELB_LIST; do
-    wait_for_state "elb" $INSTANCE_ID "InService" $elb
-    if [ $? != 0 ]; then
-        error_exit "Failed waiting for $INSTANCE_ID to return to $elb"
-    fi
-done
-
-msg "Finished $(basename $0) at $(/bin/date "+%F %T")"
-
-end_sec=$(/bin/date +%s.%N)
-elapsed_seconds=$(echo "$end_sec - $start_sec" | /usr/bin/bc)
-
-msg "Elapsed time: $elapsed_seconds"

scripts/stop_celery.py

@@ -1,82 +0,0 @@
-#!/usr/bin/env python3
-"""
-
-Scipt used to stop celery in AWS environments.
-This is used from upstart to issue a TERM signal to the master celery process.
-
-This will then allow the worker threads to stop, after completing
-whatever tasks that are in flight.
-
-Note the script blocks for up to 15minutes, which is long enough to allow our
-longest possible task to complete. If it can return quicker it will.
-
-Usage:
-    ./stop_celery.py <celery_pid_file>
-
-Example:
-    ./stop_celery.py /tmp/celery.pid
-"""
-
-import os
-from docopt import docopt
-import re
-import subprocess
-from time import sleep
-
-
-def strip_white_space(from_this):
-    return re.sub(r'\s+', '', from_this)
-
-
-def get_pid_from_file(filename):
-    """
-    Open the file which MUST contain only the PID of the master celery process.
-    This is written to disk by the start celery command issued by upstart
-    """
-    with open(filename) as f:
-        celery_pid = f.read()
-    return strip_white_space(celery_pid)
-
-
-def issue_term_signal_to_pid(pid, celery_pid_file):
-    """
-    Issues a TERM signal (15) to the master celery process.
-
-    This method attempts to print out any response from this subprocess call. However this call is generally silent.
-    """
-    print("Trying to stop ", celery_pid_file)
-    result = subprocess.Popen(['kill', '-15', pid], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    for line in result.stdout.readlines():
-        print(line.rstrip())
-    for line in result.stderr.readlines():
-        print(line.rstrip())
-
-
-def pid_still_running(pid):
-    """
-    uses the proc filesystem to identify if the celery master pid is still around.
-
-    Once the process stops this file no longer exists. Slim possibilty of a race condition here.
-    """
-    return os.path.exists("/proc/" + pid)
-
-
-if __name__ == "__main__":
-    arguments = docopt(__doc__)
-    celery_pid_file = arguments['<celery_pid_file>']
-
-    celery_pid = get_pid_from_file(celery_pid_file)
-
-    issue_term_signal_to_pid(celery_pid, celery_pid_file)
-
-    """
-    Blocking loop to check for the still running process.
-    5 seconds between loops
-    180 loops
-    Maximum block time of 900 seconds (15 minutes)
-    """
-    iteration = 0
-    while pid_still_running(celery_pid) and iteration < 180:
-        print("[", celery_pid_file, "] waited for ", iteration * 5, " secs")
-        sleep(5)
-        iteration += 1