diff --git a/appspec.yml b/appspec.yml index c6a713089..811a480cb 100644 --- a/appspec.yml +++ b/appspec.yml @@ -14,14 +14,24 @@ hooks: runas: root timeout: 300 ApplicationStart: - - + - location: scripts/aws_start_app.sh runas: root timeout: 300 - ApplicationStop: + - + location: scripts/register_with_elb.sh + runas: root + timeout: 300 + ApplicationStop: + - + location: scripts/deregister_from_elb.sh + runas: root + timeout: 300 - location: scripts/aws_stop_app.sh runas: root timeout: 300 os: linux version: 0.0 + + diff --git a/scripts/common_functions.sh b/scripts/common_functions.sh new file mode 100644 index 000000000..469d22e6c --- /dev/null +++ b/scripts/common_functions.sh @@ -0,0 +1,347 @@ +#!/bin/bash +# +# Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://aws.amazon.com/apache2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. + +# ELB_LIST defines which Elastic Load Balancers this instance should be part of. +# The elements in ELB_LIST should be seperated by space. +ELB_LIST="" + +# Under normal circumstances, you shouldn't need to change anything below this line. +# ----------------------------------------------------------------------------- + +export PATH="$PATH:/usr/bin:/usr/local/bin" + +# If true, all messages will be printed. If false, only fatal errors are printed. +DEBUG=true + +# Number of times to check for a resouce to be in the desired state. +WAITER_ATTEMPTS=60 + +# Number of seconds to wait between attempts for resource to be in a state. +WAITER_INTERVAL=1 + +# AutoScaling Standby features at minimum require this version to work. +MIN_CLI_VERSION='1.3.25' + +# Usage: get_instance_region +# +# Writes to STDOUT the AWS region as known by the local instance. +get_instance_region() { + if [ -z "$AWS_REGION" ]; then + AWS_REGION=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document \ + | grep -i region \ + | awk -F\" '{print $4}') + fi + + echo $AWS_REGION +} + +AWS_CLI="aws --region $(get_instance_region)" + + +# Usage: get_instance_state_asg +# +# Gets the state of the given as known by the AutoScaling group it's a part of. +# Health is printed to STDOUT and the function returns 0. Otherwise, no output and return is +# non-zero. +get_instance_state_asg() { + local instance_id=$1 + + local state=$($AWS_CLI autoscaling describe-auto-scaling-instances \ + --instance-ids $instance_id \ + --query "AutoScalingInstances[?InstanceId == \`$instance_id\`].LifecycleState | [0]" \ + --output text) + if [ $? != 0 ]; then + return 1 + else + echo $state + return 0 + fi +} + +reset_waiter_timeout() { + local elb=$1 + local state_name=$2 + + if [ "$state_name" == "InService" ]; then + + # Wait for a health check to succeed + local timeout=$($AWS_CLI elb describe-load-balancers \ + --load-balancer-name $elb \ + --query 'LoadBalancerDescriptions[0].HealthCheck.Timeout') + + elif [ "$state_name" == "OutOfService" ]; then + + # If connection draining is enabled, wait for connections to drain + local draining_values=$($AWS_CLI elb describe-load-balancer-attributes \ + --load-balancer-name $elb \ + --query 'LoadBalancerAttributes.ConnectionDraining.[Enabled,Timeout]' \ + --output text) + local draining_enabled=$(echo $draining_values | awk '{print $1}') + local timeout=$(echo $draining_values | awk '{print $2}') + + if [ "$draining_enabled" != "True" ]; then + timeout=0 + fi + + else + msg "Unknown state name, '$state_name'"; + return 1; + fi + + # Base register/deregister action may take up to about 30 seconds + timeout=$((timeout + 30)) + + WAITER_ATTEMPTS=$((timeout / WAITER_INTERVAL)) +} + +# Usage: wait_for_state [ELB name] +# +# Waits for the state of to be in as seen by . Returns 0 if +# it successfully made it to that state; non-zero if not. By default, checks $WAITER_ATTEMPTS +# times, every $WAITER_INTERVAL seconds. If giving an [ELB name] to check under, these are reset +# to that ELB's timeout values. +wait_for_state() { + local service=$1 + local instance_id=$2 + local state_name=$3 + local elb=$4 + + local instance_state_cmd + if [ "$service" == "elb" ]; then + instance_state_cmd="get_instance_health_elb $instance_id $elb" + reset_waiter_timeout $elb $state_name + if [ $? != 0 ]; then + error_exit "Failed resetting waiter timeout for $elb" + fi + elif [ "$service" == "autoscaling" ]; then + instance_state_cmd="get_instance_state_asg $instance_id" + else + msg "Cannot wait for instance state; unknown service type, '$service'" + return 1 + fi + + msg "Checking $WAITER_ATTEMPTS times, every $WAITER_INTERVAL seconds, for instance $instance_id to be in state $state_name" + + local instance_state=$($instance_state_cmd) + local count=1 + + msg "Instance is currently in state: $instance_state" + while [ "$instance_state" != "$state_name" ]; do + if [ $count -ge $WAITER_ATTEMPTS ]; then + local timeout=$(($WAITER_ATTEMPTS * $WAITER_INTERVAL)) + msg "Instance failed to reach state, $state_name within $timeout seconds" + return 1 + fi + + sleep $WAITER_INTERVAL + + instance_state=$($instance_state_cmd) + count=$(($count + 1)) + msg "Instance is currently in state: $instance_state" + done + + return 0 +} + +# Usage: get_instance_health_elb +# +# Gets the health of the given as known by . If it's a valid health +# status (one of InService|OutOfService|Unknown), then the health is printed to STDOUT and the +# function returns 0. Otherwise, no output and return is non-zero. +get_instance_health_elb() { + local instance_id=$1 + local elb_name=$2 + + msg "Checking status of instance '$instance_id' in load balancer '$elb_name'" + + # If describe-instance-health for this instance returns an error, then it's not part of + # this ELB. But, if the call was successful let's still double check that the status is + # valid. + local instance_status=$($AWS_CLI elb describe-instance-health \ + --load-balancer-name $elb_name \ + --instances $instance_id \ + --query 'InstanceStates[].State' \ + --output text 2>/dev/null) + + if [ $? == 0 ]; then + case "$instance_status" in + InService|OutOfService|Unknown) + echo -n $instance_status + return 0 + ;; + *) + msg "Instance '$instance_id' not part of ELB '$elb_name'" + return 1 + esac + fi +} + +# Usage: validate_elb +# +# Validates that the Elastic Load Balancer with name exists, is describable, and +# contains as one of its instances. +# +# If any of these checks are false, the function returns non-zero. +validate_elb() { + local instance_id=$1 + local elb_name=$2 + + # Get the list of active instances for this LB. + local elb_instances=$($AWS_CLI elb describe-load-balancers \ + --load-balancer-name $elb_name \ + --query 'LoadBalancerDescriptions[*].Instances[*].InstanceId' \ + --output text) + if [ $? != 0 ]; then + msg "Couldn't describe ELB instance named '$elb_name'" + return 1 + fi + + msg "Checking health of '$instance_id' as known by ELB '$elb_name'" + local instance_health=$(get_instance_health_elb $instance_id $elb_name) + if [ $? != 0 ]; then + return 1 + fi + + return 0 +} + +# Usage: get_elb_list +# +# Ensures that this instance is related to the named ELB. After execution, the variable +# "ELB_LIST" will contain the list of load balancers for the given instance. +# +# If the given instance ID isn't found registered to any ELBs, the function returns non-zero +get_elb_list() { + local instance_id=$1 + local required_elb=$2 + local elb_list="" + + msg "Looking up from ELB list" + local all_balancers=$($AWS_CLI elb describe-load-balancers \ + --query LoadBalancerDescriptions[*].LoadBalancerName \ + --output text | sed -e $'s/\t/ /g') + + if [[ $all_balancers =~ $required_elb ]] + then + local instance_health + instance_health=$(get_instance_health_elb $instance_id $required_elb) + if [ $? == 0 ]; then + elb_list="$elb_list $required_elb" + fi + fi + + if [ -z "$elb_list" ]; then + return 1 + else + msg "Got load balancer list of: $elb_list" + ELB_LIST=$elb_list + return 0 + fi +} + +# Usage: deregister_instance +# +# Deregisters from . +deregister_instance() { + local instance_id=$1 + local elb_name=$2 + + $AWS_CLI elb deregister-instances-from-load-balancer \ + --load-balancer-name $elb_name \ + --instances $instance_id 1> /dev/null + + return $? +} + +# Usage: register_instance +# +# Registers to . +register_instance() { + local instance_id=$1 + local elb_name=$2 + + $AWS_CLI elb register-instances-with-load-balancer \ + --load-balancer-name $elb_name \ + --instances $instance_id 1> /dev/null + + return $? +} + +# Usage: check_cli_version [version-to-check] [desired version] +# +# Without any arguments, checks that the installed version of the AWS CLI is at least at version +# $MIN_CLI_VERSION. Returns non-zero if the version is not high enough. +check_cli_version() { + if [ -z $1 ]; then + version=$($AWS_CLI --version 2>&1 | cut -f1 -d' ' | cut -f2 -d/) + else + version=$1 + fi + + if [ -z "$2" ]; then + min_version=$MIN_CLI_VERSION + else + min_version=$2 + fi + + x=$(echo $version | cut -f1 -d.) + y=$(echo $version | cut -f2 -d.) + z=$(echo $version | cut -f3 -d.) + + min_x=$(echo $min_version | cut -f1 -d.) + min_y=$(echo $min_version | cut -f2 -d.) + min_z=$(echo $min_version | cut -f3 -d.) + + msg "Checking minimum required CLI version (${min_version}) against installed version ($version)" + + if [ $x -lt $min_x ]; then + return 1 + elif [ $y -lt $min_y ]; then + return 1 + elif [ $y -gt $min_y ]; then + return 0 + elif [ $z -ge $min_z ]; then + return 0 + else + return 1 + fi +} + +# Usage: msg +# +# Writes to STDERR only if $DEBUG is true, otherwise has no effect. +msg() { + local message=$1 + $DEBUG && echo $message 1>&2 +} + +# Usage: error_exit +# +# Writes to STDERR as a "fatal" and immediately exits the currently running script. +error_exit() { + local message=$1 + + echo "[FATAL] $message" 1>&2 + exit 1 +} + +# Usage: get_instance_id +# +# Writes to STDOUT the EC2 instance ID for the local instance. Returns non-zero if the local +# instance metadata URL is inaccessible. +get_instance_id() { + curl -s http://169.254.169.254/latest/meta-data/instance-id + return $? +} diff --git a/scripts/deregister_from_elb.sh b/scripts/deregister_from_elb.sh new file mode 100755 index 000000000..25650196b --- /dev/null +++ b/scripts/deregister_from_elb.sh @@ -0,0 +1,69 @@ +#!/bin/bash +# +# Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://aws.amazon.com/apache2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. + +. $(dirname $0)/common_functions.sh + +msg "Running AWS CLI with region: $(get_instance_region)" + +# get this instance's ID +INSTANCE_ID=$(get_instance_id) +if [ $? != 0 -o -z "$INSTANCE_ID" ]; then + error_exit "Unable to get this instance's ID; cannot continue." +fi + +# Get current time +msg "Started $(basename $0) at $(/bin/date "+%F %T")" +start_sec=$(/bin/date +%s.%N) + +msg "Getting relevant load balancer" +get_elb_list $INSTANCE_ID "notify-admin-elb" + +msg "Checking that user set at least one load balancer" +if test -z "$ELB_LIST"; then + error_exit "Must have at least one load balancer to deregister from" +fi + +# Loop through all LBs the user set, and attempt to deregister this instance from them. +for elb in $ELB_LIST; do + msg "Checking validity of load balancer named '$elb'" + validate_elb $INSTANCE_ID $elb + if [ $? != 0 ]; then + msg "Error validating $elb; cannot continue with this LB" + continue + fi + + msg "Deregistering $INSTANCE_ID from $elb" + deregister_instance $INSTANCE_ID $elb + + if [ $? != 0 ]; then + error_exit "Failed to deregister instance $INSTANCE_ID from ELB $elb" + fi +done + +# Wait for all Deregistrations to finish +msg "Waiting for instance to de-register from its load balancers" +for elb in $ELB_LIST; do + wait_for_state "elb" $INSTANCE_ID "OutOfService" $elb + if [ $? != 0 ]; then + error_exit "Failed waiting for $INSTANCE_ID to leave $elb" + fi +done + +msg "Finished $(basename $0) at $(/bin/date "+%F %T")" + +end_sec=$(/bin/date +%s.%N) +elapsed_seconds=$(echo "$end_sec - $start_sec" | /usr/bin/bc) + +msg "Elapsed time: $elapsed_seconds" diff --git a/scripts/register_with_elb.sh b/scripts/register_with_elb.sh new file mode 100755 index 000000000..9385716d4 --- /dev/null +++ b/scripts/register_with_elb.sh @@ -0,0 +1,70 @@ +#!/bin/bash +# +# Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://aws.amazon.com/apache2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. + +. $(dirname $0)/common_functions.sh + +msg "Running AWS CLI with region: $(get_instance_region)" + +# get this instance's ID +INSTANCE_ID=$(get_instance_id) +if [ $? != 0 -o -z "$INSTANCE_ID" ]; then + error_exit "Unable to get this instance's ID; cannot continue." +fi + +# Get current time +msg "Started $(basename $0) at $(/bin/date "+%F %T")" +start_sec=$(/bin/date +%s.%N) + +msg "Getting relevant load balancer" +get_elb_list $INSTANCE_ID "notify-admin-elb" + + +msg "Checking that user set at least one load balancer" +if test -z "$ELB_LIST"; then + error_exit "Must have at least one load balancer to register to" +fi + +# Loop through all LBs the user set, and attempt to register this instance to them. +for elb in $ELB_LIST; do + msg "Checking validity of load balancer named '$elb'" + validate_elb $INSTANCE_ID $elb + if [ $? != 0 ]; then + msg "Error validating $elb; cannot continue with this LB" + continue + fi + + msg "Registering $INSTANCE_ID to $elb" + register_instance $INSTANCE_ID $elb + + if [ $? != 0 ]; then + error_exit "Failed to register instance $INSTANCE_ID from ELB $elb" + fi +done + +# Wait for all Registrations to finish +msg "Waiting for instance to register to its load balancers" +for elb in $ELB_LIST; do + wait_for_state "elb" $INSTANCE_ID "InService" $elb + if [ $? != 0 ]; then + error_exit "Failed waiting for $INSTANCE_ID to return to $elb" + fi +done + +msg "Finished $(basename $0) at $(/bin/date "+%F %T")" + +end_sec=$(/bin/date +%s.%N) +elapsed_seconds=$(echo "$end_sec - $start_sec" | /usr/bin/bc) + +msg "Elapsed time: $elapsed_seconds" diff --git a/tests/conftest.py b/tests/conftest.py index 45fbfb008..2ad9c78b6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -77,6 +77,7 @@ def mock_get_live_service(mocker, api_user_active): active=False, restricted=False) return {'data': service} + return mocker.patch('app.service_api_client.get_service', side_effect=_get) @@ -230,6 +231,7 @@ def mock_get_template_version(mocker, fake_uuid, user=None): version=version ) return {'data': template_version} + return mocker.patch( 'app.service_api_client.get_service_template', side_effect=_get @@ -249,6 +251,7 @@ def mock_get_template_versions(mocker, fake_uuid, user=None): version=1 ) return {'data': [template_version]} + return mocker.patch( 'app.service_api_client.get_service_template_versions', side_effect=_get @@ -528,6 +531,7 @@ def mock_get_user(mocker, user=None): def _get_user(id_): user.id = id_ return user + return mocker.patch( 'app.user_api_client.get_user', side_effect=_get_user) @@ -557,6 +561,7 @@ def mock_get_user_by_email(mocker, user=None): def _get_user(email_address): user._email_address = email_address return user + return mocker.patch('app.user_api_client.get_user_by_email', side_effect=_get_user) @@ -570,15 +575,16 @@ def mock_get_user_with_permissions(mocker, api_user_active): def _get_user(id): api_user_active._permissions[''] = ['manage_users', 'manage_templates', 'manage_settings'] return api_user_active + return mocker.patch( 'app.user_api_client.get_user', side_effect=_get_user) @pytest.fixture(scope='function') def mock_dont_get_user_by_email(mocker): - def _get_user(email_address): return None + return mocker.patch( 'app.user_api_client.get_user_by_email', side_effect=_get_user, @@ -625,6 +631,7 @@ def mock_get_user_by_email_not_found(mocker, api_user_active): resp_mock = Mock(status_code=404, json=json_mock) http_error = HTTPError(response=resp_mock, message="Default message") raise http_error + return mocker.patch( 'app.user_api_client.get_user_by_email', side_effect=_get_user) @@ -634,6 +641,7 @@ def mock_get_user_by_email_not_found(mocker, api_user_active): def mock_verify_password(mocker): def _verify_password(user, password): return True + return mocker.patch( 'app.user_api_client.verify_password', side_effect=_verify_password) @@ -641,9 +649,9 @@ def mock_verify_password(mocker): @pytest.fixture(scope='function') def mock_update_user(mocker): - def _update(user): return user + return mocker.patch('app.user_api_client.update_user', side_effect=_update) @@ -664,7 +672,6 @@ def mock_get_all_users_from_api(mocker): @pytest.fixture(scope='function') def mock_create_api_key(mocker): - def _create(service_id, key_name): import uuid return {'data': str(generate_uuid())} @@ -735,6 +742,7 @@ def mock_send_verify_email(mocker): def mock_check_verify_code(mocker): def _verify(user_id, code, code_type): return True, '' + return mocker.patch( 'app.user_api_client.check_verify_code', side_effect=_verify) @@ -744,6 +752,7 @@ def mock_check_verify_code(mocker): def mock_check_verify_code_code_not_found(mocker): def _verify(user_id, code, code_type): return False, 'Code not found' + return mocker.patch( 'app.user_api_client.check_verify_code', side_effect=_verify) @@ -753,6 +762,7 @@ def mock_check_verify_code_code_not_found(mocker): def mock_check_verify_code_code_expired(mocker): def _verify(user_id, code, code_type): return False, 'Code has expired' + return mocker.patch( 'app.user_api_client.check_verify_code', side_effect=_verify) @@ -774,6 +784,7 @@ def mock_create_job(mocker, job_data): job_data['file_name'] = '{}.csv'.format(job_id) job_data['notification_count'] = notification_count return job_data + return mocker.patch('app.job_api_client.create_job', side_effect=_create) @@ -783,6 +794,7 @@ def mock_get_job(mocker, job_data): job_data['id'] = job_id job_data['service'] = service_id return {"data": job_data} + return mocker.patch('app.job_api_client.get_job', side_effect=_get_job) @@ -794,6 +806,7 @@ def mock_get_jobs(mocker): job_data = job_json_with_created_by(service_id=service_id) data.append(job_data) return {"data": data} + return mocker.patch('app.job_api_client.get_job', side_effect=_get_jobs) @@ -807,6 +820,7 @@ def mock_get_notifications(mocker): status=None, limit_days=None): return notification_json(service_id) + return mocker.patch( 'app.notification_api_client.get_notifications_for_service', side_effect=_get_notifications @@ -822,6 +836,7 @@ def mock_get_notifications_with_previous_next(mocker): status=None, limit_days=None): return notification_json(service_id, with_links=True) + return mocker.patch( 'app.notification_api_client.get_notifications_for_service', side_effect=_get_notifications @@ -832,6 +847,7 @@ def mock_get_notifications_with_previous_next(mocker): def mock_has_permissions(mocker): def _has_permission(permissions=None, any_=False, admin_override=False): return True + return mocker.patch( 'app.notify_client.user_api_client.User.has_permissions', side_effect=_has_permission) @@ -857,6 +873,7 @@ def mock_get_users_by_service(mocker): 'email_address': 'notify@digital.cabinet-office.gov.uk', 'failed_login_count': 0}] return [User(data[0])] + return mocker.patch('app.user_api_client.get_users_for_service', side_effect=_get_users_for_service, autospec=True) @@ -864,6 +881,7 @@ def mock_get_users_by_service(mocker): def mock_s3_upload(mocker): def _upload(service_id, filedata, region): return fake_uuid() + return mocker.patch('app.main.views.send.s3upload', side_effect=_upload) @@ -885,7 +903,6 @@ def sample_invited_user(mocker, sample_invite): @pytest.fixture(scope='function') def mock_create_invite(mocker, sample_invite): - def _create_invite(from_user, service_id, email_address, permissions): sample_invite['from_user'] = from_user sample_invite['service'] = service_id @@ -893,6 +910,7 @@ def mock_create_invite(mocker, sample_invite): sample_invite['status'] = 'pending' sample_invite['permissions'] = permissions return InvitedUser(**sample_invite) + return mocker.patch('app.invite_api_client.create_invite', side_effect=_create_invite) @@ -907,6 +925,7 @@ def mock_get_invites_for_service(mocker, service_one, sample_invite): invite['email_address'] = 'user_{}@testnotify.gov.uk'.format(i) data.append(InvitedUser(**invite)) return data + return mocker.patch('app.invite_api_client.get_invites_for_service', side_effect=_get_invites) @@ -914,6 +933,7 @@ def mock_get_invites_for_service(mocker, service_one, sample_invite): def mock_check_invite_token(mocker, sample_invite): def _check_token(token): return InvitedUser(**sample_invite) + return mocker.patch('app.invite_api_client.check_token', side_effect=_check_token) @@ -921,6 +941,7 @@ def mock_check_invite_token(mocker, sample_invite): def mock_accept_invite(mocker, sample_invite): def _accept(service_id, invite_id): return InvitedUser(**sample_invite) + return mocker.patch('app.invite_api_client.accept_invite', side_effect=_accept) @@ -928,6 +949,7 @@ def mock_accept_invite(mocker, sample_invite): def mock_add_user_to_service(mocker, service_one, api_user_active): def _add_user(service_id, user_id, permissions): return api_user_active + return mocker.patch('app.user_api_client.add_user_to_service', side_effect=_add_user) @@ -965,11 +987,10 @@ def mock_get_template_statistics(mocker, service_one, fake_uuid): @pytest.fixture(scope='function') def mock_get_usage(mocker, service_one, fake_uuid): - def _get_usage(service_id): return {'data': { - "sms_count": 123, - "email_count": 456 + "sms_count": 123, + "email_count": 456 }} return mocker.patch( @@ -978,7 +999,6 @@ def mock_get_usage(mocker, service_one, fake_uuid): @pytest.fixture(scope='function') def mock_events(mocker): - def _create_event(event_type, event_data): return {'some': 'data'}