Merge pull request #2503 from alphagov/revert-2498-zdt-deploy

Revert "Zero downtime deploy"
This commit is contained in:
Leo Hemsted
2019-05-15 14:01:15 +01:00
committed by GitHub
3 changed files with 33 additions and 26 deletions

View File

@@ -231,18 +231,25 @@ cf-deploy: scripts/statsd_exporter ## Deploys the app to Cloud Foundry
$(if ${CF_APP},,$(error Must specify CF_APP))
cf target -o ${CF_ORG} -s ${CF_SPACE}
@cf app --guid ${CF_APP} || exit 1
cf rename ${CF_APP} ${CF_APP}-rollback
cf push ${CF_APP} -f <(make -s generate-manifest)
cf scale -i $$(cf curl /v2/apps/$$(cf app --guid ${CF_APP}-rollback) | jq -r ".entity.instances" 2>/dev/null || echo "1") ${CF_APP}
cf stop ${CF_APP}-rollback
# sleep for 10 seconds to try and make sure that all worker threads (either web api or celery) have finished before we delete
# when we delete the DB is unbound from the app, which can cause "permission denied for relation" psycopg2 errors.
sleep 10
# cancel any existing deploys to ensure we can apply manifest (if a deploy is in progress you'll see ScaleDisabledDuringDeployment)
cf v3-cancel-zdt-push ${CF_APP} || true
cf v3-apply-manifest ${CF_APP} -f <(make -s generate-manifest)
cf v3-zdt-push ${CF_APP} --wait-for-deploy-complete # fails after 5 mins if deploy doesn't work
# get the new GUID, and find all crash events for that. If there were any crashes we will abort the deploy.
[ $$(cf curl "/v2/events?q=type:app.crash&q=actee:$$(cf app --guid ${CF_APP})" | jq ".total_results") -eq 0 ]
cf delete -f ${CF_APP}-rollback
.PHONY: cf-deploy-api-db-migration
cf-deploy-api-db-migration:
$(if ${CF_SPACE},,$(error Must specify CF_SPACE))
cf target -o ${CF_ORG} -s ${CF_SPACE}
cf unbind-service notify-api-db-migration notify-db
cf unbind-service notify-api-db-migration notify-config
cf unbind-service notify-api-db-migration notify-aws
cf push notify-api-db-migration -f <(make -s CF_APP=notify-api-db-migration generate-manifest)
cf run-task notify-api-db-migration "flask db upgrade" --name api_db_migration
@@ -253,7 +260,10 @@ cf-check-api-db-migration-task: ## Get the status for the last notify-api-db-mig
.PHONY: cf-rollback
cf-rollback: ## Rollbacks the app to the previous release
$(if ${CF_APP},,$(error Must specify CF_APP))
cf v3-cancel-zdt-push ${CF_APP}
@cf app --guid ${CF_APP}-rollback || exit 1
@[ $$(cf curl /v2/apps/`cf app --guid ${CF_APP}-rollback` | jq -r ".entity.state") = "STARTED" ] || (echo "Error: rollback is not possible because ${CF_APP}-rollback is not in a started state" && exit 1)
cf delete -f ${CF_APP} || true
cf rename ${CF_APP}-rollback ${CF_APP}
.PHONY: cf-push
cf-push: scripts/statsd_exporter

View File

@@ -156,9 +156,7 @@ This means that the first deployment of your app must happen manually.
To do this:
1. Run `cf target -s <cf-space>; cf v3-create-app <APP_NAME>`
1. Repeat for each cf environment
1. Ensure your code is backwards compatible
1. From the root of this repo run `CF_APP=<APP_NAME> make <cf-space> cf-push`
Once this is done, you can push your deployment changes to jenkins to have your app deployed on every deployment.
To delete an unused app, repeat the steps in reverse order. After you are no longer scaling, deploying, or alerting on that box, it's safe to run `cf stop <APP_NAME>`. Then, wait for a minute to make sure all tasks have finished executing, and after one last check to make sure notify still works, delete with `cf delete <APP_NAME>`

View File

@@ -1,10 +1,9 @@
{%- set app_vars = {
'notify-api': {'NOTIFY_APP_NAME': 'api', 'disk_quota': '2G', 'sqlalchemy_pool_size': 20, 'routes': {
'preview': ['api.notify.works'],
'staging': ['api.staging-notify.works'],
'production': ['api.notifications.service.gov.uk'],
},
'healthcheck-endpoint': '/_status',
'preview': ['notify-api-preview.cloudapps.digital', 'api.notify.works'],
'staging': ['notify-api-staging.cloudapps.digital', 'api.staging-notify.works'],
'production': ['notify-api-production.cloudapps.digital', 'api.notifications.service.gov.uk'],
}
},
'notify-api-db-migration': {'NOTIFY_APP_NAME': 'api', 'instances': 0},
@@ -34,24 +33,24 @@ applications:
memory: {{ app.get('memory', '1G') }}
disk_quota: {{ app.get('disk_quota', '1G')}}
{% if 'routes' in app -%}
routes:
{%- for route in app.get('routes', {}).get(environment, []) %}
{%- for route in app['routes'][environment] %}
- route: {{ route }}
{%- endfor%}
{%- endfor -%}
{%- elif environment in app.get('local_statsd', []) -%}
health-check-type: none
routes:
- route: {{ CF_APP }}-{{ environment }}.cloudapps.digital
{% if 'healthcheck-endpoint' in app %}
health-check-type: http
health-check-http-endpoint: {{ app['healthcheck-endpoint'] }}
{% else %}
health-check-type: process
{%- else -%}
health-check-type: none
no-route: true
{% endif %}
services:
- notify-db
- logit-ssl-syslog-drain
{% if environment in app.get('local_statsd', []) -%}
- notify-prometheus
{% endif %}
{% if environment in app.get('local_statsd', []) %}- notify-prometheus{% endif %}
env:
NOTIFY_APP_NAME: {{ app.get('NOTIFY_APP_NAME', CF_APP.replace('notify-', '')) }}