From 53806f168d5443c153eba0a194850ff98d07ff15 Mon Sep 17 00:00:00 2001 From: Leo Hemsted Date: Tue, 30 Apr 2019 17:10:56 +0100 Subject: [PATCH 1/5] add routes for all apps all apps get a route assigned when using v3-zdt-push. > By default, the web process has a route and one instance. Other processes have zero instances by default. ([source](https://docs.cloudfoundry.org/devguide/multiple-processes.html)) When we push apps to multiple environments they need different routes or the second push will fail, so this means that we need to define routes ourselves for every app. We're also manually flagging the health-check as either "http" or "process" - http for the api, process for all others. --- manifest.yml.j2 | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/manifest.yml.j2 b/manifest.yml.j2 index 2d94b5548..90745b554 100644 --- a/manifest.yml.j2 +++ b/manifest.yml.j2 @@ -1,9 +1,10 @@ {%- set app_vars = { 'notify-api': {'NOTIFY_APP_NAME': 'api', 'disk_quota': '2G', 'sqlalchemy_pool_size': 20, 'routes': { - 'preview': ['notify-api-preview.cloudapps.digital', 'api.notify.works'], - 'staging': ['notify-api-staging.cloudapps.digital', 'api.staging-notify.works'], - 'production': ['notify-api-production.cloudapps.digital', 'api.notifications.service.gov.uk'], - } + 'preview': ['api.notify.works'], + 'staging': ['api.staging-notify.works'], + 'production': ['api.notifications.service.gov.uk'], + }, + 'healthcheck-endpoint': '/_status', }, 'notify-api-db-migration': {'NOTIFY_APP_NAME': 'api', 'instances': 0}, @@ -33,24 +34,24 @@ applications: memory: {{ app.get('memory', '1G') }} disk_quota: {{ app.get('disk_quota', '1G')}} - {% if 'routes' in app -%} routes: - {%- for route in app['routes'][environment] %} + {%- for route in app.get('routes', {}).get(environment, []) %} - route: {{ route }} - {%- endfor -%} - {%- elif environment in app.get('local_statsd', []) -%} - health-check-type: none - routes: + {%- endfor%} - route: {{ CF_APP }}-{{ environment }}.cloudapps.digital - {%- else -%} - health-check-type: none - no-route: true + {% if 'healthcheck-endpoint' in app %} + health-check-type: http + health-check-http-endpoint: {{ app['healthcheck-endpoint'] }} + {% else %} + health-check-type: process {% endif %} services: - notify-db - logit-ssl-syslog-drain - {% if environment in app.get('local_statsd', []) %}- notify-prometheus{% endif %} + {% if environment in app.get('local_statsd', []) -%} + - notify-prometheus + {% endif %} env: NOTIFY_APP_NAME: {{ app.get('NOTIFY_APP_NAME', CF_APP.replace('notify-', '')) }} From b2f87378c3dca5c83a97030f5fecce9225c9487a Mon Sep 17 00:00:00 2001 From: Leo Hemsted Date: Thu, 2 May 2019 11:02:49 +0100 Subject: [PATCH 2/5] update manifest file to use zdt-push this way we keep db bindings etc, and avoid accidentally dropping connections. --- Makefile | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/Makefile b/Makefile index 79ab98583..b52ff4b1b 100644 --- a/Makefile +++ b/Makefile @@ -231,25 +231,18 @@ cf-deploy: scripts/statsd_exporter ## Deploys the app to Cloud Foundry $(if ${CF_APP},,$(error Must specify CF_APP)) cf target -o ${CF_ORG} -s ${CF_SPACE} @cf app --guid ${CF_APP} || exit 1 - cf rename ${CF_APP} ${CF_APP}-rollback - cf push ${CF_APP} -f <(make -s generate-manifest) - cf scale -i $$(cf curl /v2/apps/$$(cf app --guid ${CF_APP}-rollback) | jq -r ".entity.instances" 2>/dev/null || echo "1") ${CF_APP} - cf stop ${CF_APP}-rollback - # sleep for 10 seconds to try and make sure that all worker threads (either web api or celery) have finished before we delete - # when we delete the DB is unbound from the app, which can cause "permission denied for relation" psycopg2 errors. - sleep 10 - # get the new GUID, and find all crash events for that. If there were any crashes we will abort the deploy. - [ $$(cf curl "/v2/events?q=type:app.crash&q=actee:$$(cf app --guid ${CF_APP})" | jq ".total_results") -eq 0 ] - cf delete -f ${CF_APP}-rollback + # cancel any existing deploys to ensure we can apply manifest (if a deploy is in progress you'll see ScaleDisabledDuringDeployment) + cf v3-cancel-zdt-push ${CF_APP} || true + + cf v3-apply-manifest ${CF_APP} -f <(make -s generate-manifest) + cf v3-zdt-push ${CF_APP} --wait-for-deploy-complete # fails after 5 mins if deploy doesn't work + .PHONY: cf-deploy-api-db-migration cf-deploy-api-db-migration: $(if ${CF_SPACE},,$(error Must specify CF_SPACE)) cf target -o ${CF_ORG} -s ${CF_SPACE} - cf unbind-service notify-api-db-migration notify-db - cf unbind-service notify-api-db-migration notify-config - cf unbind-service notify-api-db-migration notify-aws cf push notify-api-db-migration -f <(make -s CF_APP=notify-api-db-migration generate-manifest) cf run-task notify-api-db-migration "flask db upgrade" --name api_db_migration @@ -260,10 +253,7 @@ cf-check-api-db-migration-task: ## Get the status for the last notify-api-db-mig .PHONY: cf-rollback cf-rollback: ## Rollbacks the app to the previous release $(if ${CF_APP},,$(error Must specify CF_APP)) - @cf app --guid ${CF_APP}-rollback || exit 1 - @[ $$(cf curl /v2/apps/`cf app --guid ${CF_APP}-rollback` | jq -r ".entity.state") = "STARTED" ] || (echo "Error: rollback is not possible because ${CF_APP}-rollback is not in a started state" && exit 1) - cf delete -f ${CF_APP} || true - cf rename ${CF_APP}-rollback ${CF_APP} + cf v3-cancel-zdt-push ${CF_APP} .PHONY: cf-push cf-push: scripts/statsd_exporter From 39c8e1ede4fe794a19c4e90e3ca948131f84628f Mon Sep 17 00:00:00 2001 From: Leo Hemsted Date: Thu, 9 May 2019 14:50:38 +0100 Subject: [PATCH 3/5] update readme --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3688dee86..e7676a4fa 100644 --- a/README.md +++ b/README.md @@ -156,7 +156,9 @@ This means that the first deployment of your app must happen manually. To do this: -1. Ensure your code is backwards compatible -1. From the root of this repo run `CF_APP= make cf-push` +1. Run `cf target -s ; cf v3-create-app ` +1. Repeat for each cf environment Once this is done, you can push your deployment changes to jenkins to have your app deployed on every deployment. + +To delete an unused app, repeat the steps in reverse order. After you are no longer scaling, deploying, or alerting on that box, it's safe to run `cf stop `. Then, wait for a minute to make sure all tasks have finished executing, and after one last check to make sure notify still works, delete with `cf delete ` From 84c35cbf6784ef89a12547d19d3c2bc0b0547165 Mon Sep 17 00:00:00 2001 From: Katie Smith Date: Fri, 10 May 2019 11:47:42 +0100 Subject: [PATCH 4/5] Add request_to_go_live_notes column to Organisation table This column will be used to stores extra notes that should go in the Zendesk ticket when a service belonging to that organisation requests to go live. --- app/models.py | 2 ++ migrations/versions/0290_org_go_live_notes.py | 21 +++++++++++++++++++ tests/app/organisation/test_rest.py | 2 ++ 3 files changed, 25 insertions(+) create mode 100644 migrations/versions/0290_org_go_live_notes.py diff --git a/app/models.py b/app/models.py index d0a6a3d25..acefd68eb 100644 --- a/app/models.py +++ b/app/models.py @@ -341,6 +341,7 @@ class Organisation(db.Model): agreement_signed_version = db.Column(db.Float, nullable=True) crown = db.Column(db.Boolean, nullable=True) organisation_type = db.Column(db.String(255), nullable=True) + request_to_go_live_notes = db.Column(db.Text) domains = db.relationship( 'Domain', @@ -376,6 +377,7 @@ class Organisation(db.Model): "domains": [ domain.domain for domain in self.domains ], + "request_to_go_live_notes": self.request_to_go_live_notes, } diff --git a/migrations/versions/0290_org_go_live_notes.py b/migrations/versions/0290_org_go_live_notes.py new file mode 100644 index 000000000..ed963bccb --- /dev/null +++ b/migrations/versions/0290_org_go_live_notes.py @@ -0,0 +1,21 @@ +""" + +Revision ID: 0290_org_go_live_notes +Revises: 0289_precompiled_for_all +Create Date: 2019-05-13 14:55:10.291781 + +""" +from alembic import op +import sqlalchemy as sa + + +revision = '0290_org_go_live_notes' +down_revision = '0289_precompiled_for_all' + + +def upgrade(): + op.add_column('organisation', sa.Column('request_to_go_live_notes', sa.Text(), nullable=True)) + + +def downgrade(): + op.drop_column('organisation', 'request_to_go_live_notes') diff --git a/tests/app/organisation/test_rest.py b/tests/app/organisation/test_rest.py index 0a70baa52..4c231e50b 100644 --- a/tests/app/organisation/test_rest.py +++ b/tests/app/organisation/test_rest.py @@ -52,6 +52,7 @@ def test_get_organisation_by_id(admin_request, notify_db_session): 'letter_branding_id', 'email_branding_id', 'domains', + 'request_to_go_live_notes', } assert response['id'] == str(org.id) assert response['name'] == 'test_org_1' @@ -64,6 +65,7 @@ def test_get_organisation_by_id(admin_request, notify_db_session): assert response['letter_branding_id'] is None assert response['email_branding_id'] is None assert response['domains'] == [] + assert response['request_to_go_live_notes'] is None def test_get_organisation_by_id_returns_domains(admin_request, notify_db_session): From 7a711cf31480056296a303df67df6dbdf63edd02 Mon Sep 17 00:00:00 2001 From: Leo Hemsted Date: Wed, 15 May 2019 13:48:40 +0100 Subject: [PATCH 5/5] Revert "Zero downtime deploy" --- Makefile | 24 +++++++++++++++++------- README.md | 6 ++---- manifest.yml.j2 | 29 ++++++++++++++--------------- 3 files changed, 33 insertions(+), 26 deletions(-) diff --git a/Makefile b/Makefile index b52ff4b1b..79ab98583 100644 --- a/Makefile +++ b/Makefile @@ -231,18 +231,25 @@ cf-deploy: scripts/statsd_exporter ## Deploys the app to Cloud Foundry $(if ${CF_APP},,$(error Must specify CF_APP)) cf target -o ${CF_ORG} -s ${CF_SPACE} @cf app --guid ${CF_APP} || exit 1 + cf rename ${CF_APP} ${CF_APP}-rollback + cf push ${CF_APP} -f <(make -s generate-manifest) + cf scale -i $$(cf curl /v2/apps/$$(cf app --guid ${CF_APP}-rollback) | jq -r ".entity.instances" 2>/dev/null || echo "1") ${CF_APP} + cf stop ${CF_APP}-rollback + # sleep for 10 seconds to try and make sure that all worker threads (either web api or celery) have finished before we delete + # when we delete the DB is unbound from the app, which can cause "permission denied for relation" psycopg2 errors. + sleep 10 - # cancel any existing deploys to ensure we can apply manifest (if a deploy is in progress you'll see ScaleDisabledDuringDeployment) - cf v3-cancel-zdt-push ${CF_APP} || true - - cf v3-apply-manifest ${CF_APP} -f <(make -s generate-manifest) - cf v3-zdt-push ${CF_APP} --wait-for-deploy-complete # fails after 5 mins if deploy doesn't work - + # get the new GUID, and find all crash events for that. If there were any crashes we will abort the deploy. + [ $$(cf curl "/v2/events?q=type:app.crash&q=actee:$$(cf app --guid ${CF_APP})" | jq ".total_results") -eq 0 ] + cf delete -f ${CF_APP}-rollback .PHONY: cf-deploy-api-db-migration cf-deploy-api-db-migration: $(if ${CF_SPACE},,$(error Must specify CF_SPACE)) cf target -o ${CF_ORG} -s ${CF_SPACE} + cf unbind-service notify-api-db-migration notify-db + cf unbind-service notify-api-db-migration notify-config + cf unbind-service notify-api-db-migration notify-aws cf push notify-api-db-migration -f <(make -s CF_APP=notify-api-db-migration generate-manifest) cf run-task notify-api-db-migration "flask db upgrade" --name api_db_migration @@ -253,7 +260,10 @@ cf-check-api-db-migration-task: ## Get the status for the last notify-api-db-mig .PHONY: cf-rollback cf-rollback: ## Rollbacks the app to the previous release $(if ${CF_APP},,$(error Must specify CF_APP)) - cf v3-cancel-zdt-push ${CF_APP} + @cf app --guid ${CF_APP}-rollback || exit 1 + @[ $$(cf curl /v2/apps/`cf app --guid ${CF_APP}-rollback` | jq -r ".entity.state") = "STARTED" ] || (echo "Error: rollback is not possible because ${CF_APP}-rollback is not in a started state" && exit 1) + cf delete -f ${CF_APP} || true + cf rename ${CF_APP}-rollback ${CF_APP} .PHONY: cf-push cf-push: scripts/statsd_exporter diff --git a/README.md b/README.md index e7676a4fa..3688dee86 100644 --- a/README.md +++ b/README.md @@ -156,9 +156,7 @@ This means that the first deployment of your app must happen manually. To do this: -1. Run `cf target -s ; cf v3-create-app ` -1. Repeat for each cf environment +1. Ensure your code is backwards compatible +1. From the root of this repo run `CF_APP= make cf-push` Once this is done, you can push your deployment changes to jenkins to have your app deployed on every deployment. - -To delete an unused app, repeat the steps in reverse order. After you are no longer scaling, deploying, or alerting on that box, it's safe to run `cf stop `. Then, wait for a minute to make sure all tasks have finished executing, and after one last check to make sure notify still works, delete with `cf delete ` diff --git a/manifest.yml.j2 b/manifest.yml.j2 index 90745b554..2d94b5548 100644 --- a/manifest.yml.j2 +++ b/manifest.yml.j2 @@ -1,10 +1,9 @@ {%- set app_vars = { 'notify-api': {'NOTIFY_APP_NAME': 'api', 'disk_quota': '2G', 'sqlalchemy_pool_size': 20, 'routes': { - 'preview': ['api.notify.works'], - 'staging': ['api.staging-notify.works'], - 'production': ['api.notifications.service.gov.uk'], - }, - 'healthcheck-endpoint': '/_status', + 'preview': ['notify-api-preview.cloudapps.digital', 'api.notify.works'], + 'staging': ['notify-api-staging.cloudapps.digital', 'api.staging-notify.works'], + 'production': ['notify-api-production.cloudapps.digital', 'api.notifications.service.gov.uk'], + } }, 'notify-api-db-migration': {'NOTIFY_APP_NAME': 'api', 'instances': 0}, @@ -34,24 +33,24 @@ applications: memory: {{ app.get('memory', '1G') }} disk_quota: {{ app.get('disk_quota', '1G')}} + {% if 'routes' in app -%} routes: - {%- for route in app.get('routes', {}).get(environment, []) %} + {%- for route in app['routes'][environment] %} - route: {{ route }} - {%- endfor%} + {%- endfor -%} + {%- elif environment in app.get('local_statsd', []) -%} + health-check-type: none + routes: - route: {{ CF_APP }}-{{ environment }}.cloudapps.digital - {% if 'healthcheck-endpoint' in app %} - health-check-type: http - health-check-http-endpoint: {{ app['healthcheck-endpoint'] }} - {% else %} - health-check-type: process + {%- else -%} + health-check-type: none + no-route: true {% endif %} services: - notify-db - logit-ssl-syslog-drain - {% if environment in app.get('local_statsd', []) -%} - - notify-prometheus - {% endif %} + {% if environment in app.get('local_statsd', []) %}- notify-prometheus{% endif %} env: NOTIFY_APP_NAME: {{ app.get('NOTIFY_APP_NAME', CF_APP.replace('notify-', '')) }}