From 8cc5d4029187bca2bb846c438968032f41fc0e31 Mon Sep 17 00:00:00 2001 From: Leo Hemsted Date: Wed, 10 Apr 2019 15:21:30 +0100 Subject: [PATCH] remove old manifest files and creation script --- Makefile | 10 --- manifest-api-base.yml | 67 ------------------- manifest-api-preview.yml | 10 --- manifest-api-production.yml | 9 --- manifest-api-sandbox.yml | 6 -- manifest-api-staging.yml | 9 --- manifest-delivery-base.yml | 107 ------------------------------- manifest-delivery-preview.yml | 5 -- manifest-delivery-production.yml | 6 -- manifest-delivery-sandbox.yml | 3 - manifest-delivery-staging.yml | 6 -- scripts/generate_manifest.py | 65 ------------------- 12 files changed, 303 deletions(-) delete mode 100644 manifest-api-base.yml delete mode 100644 manifest-api-preview.yml delete mode 100644 manifest-api-production.yml delete mode 100644 manifest-api-sandbox.yml delete mode 100644 manifest-api-staging.yml delete mode 100644 manifest-delivery-base.yml delete mode 100644 manifest-delivery-preview.yml delete mode 100644 manifest-delivery-production.yml delete mode 100644 manifest-delivery-sandbox.yml delete mode 100644 manifest-delivery-staging.yml delete mode 100755 scripts/generate_manifest.py diff --git a/Makefile b/Makefile index d312b845a..c8a182706 100644 --- a/Makefile +++ b/Makefile @@ -209,16 +209,6 @@ cf-login: ## Log in to Cloud Foundry @echo "Logging in to Cloud Foundry on ${CF_API}" @cf login -a "${CF_API}" -u ${CF_USERNAME} -p "${CF_PASSWORD}" -o "${CF_ORG}" -s "${CF_SPACE}" -.PHONY: generate-manifest-old -generate-manifest-old: - $(if ${CF_APP},,$(error Must specify CF_APP)) - $(if ${CF_SPACE},,$(error Must specify CF_SPACE)) - $(if $(shell which gpg2), $(eval export GPG=gpg2), $(eval export GPG=gpg)) - $(if ${GPG_PASSPHRASE_TXT}, $(eval export DECRYPT_CMD=echo -n $$$${GPG_PASSPHRASE_TXT} | ${GPG} --quiet --batch --passphrase-fd 0 --pinentry-mode loopback -d), $(eval export DECRYPT_CMD=${GPG} --quiet --batch -d)) - - @./scripts/generate_manifest.py ${CF_MANIFEST_FILE} \ - <(${DECRYPT_CMD} ${NOTIFY_CREDENTIALS}/credentials/${CF_SPACE}/paas/environment-variables.gpg) - .PHONY: generate-manifest generate-manifest: $(if ${CF_APP},,$(error Must specify CF_APP)) diff --git a/manifest-api-base.yml b/manifest-api-base.yml deleted file mode 100644 index 5901d45dc..000000000 --- a/manifest-api-base.yml +++ /dev/null @@ -1,67 +0,0 @@ ---- - -buildpack: python_buildpack -command: unset GUNICORN_CMD_ARGS; scripts/run_app_paas.sh gunicorn -c /home/vcap/app/gunicorn_config.py application - -services: - - notify-db - - logit-ssl-syslog-drain - -env: - NOTIFY_APP_NAME: public-api - CW_APP_NAME: api - # required by cf run-task - FLASK_APP: application.py - SQLALCHEMY_POOL_SIZE: 20 - - # Credentials variables - ADMIN_BASE_URL: null - ADMIN_CLIENT_SECRET: null - API_HOST_NAME: null - DANGEROUS_SALT: null - SECRET_KEY: null - ROUTE_SECRET_KEY_1: null - ROUTE_SECRET_KEY_2: null - CRONITOR_KEYS: null - - PERFORMANCE_PLATFORM_ENDPOINTS: null - - NOTIFICATION_QUEUE_PREFIX: null - AWS_ACCESS_KEY_ID: null - AWS_SECRET_ACCESS_KEY: null - - STATSD_PREFIX: null - - ZENDESK_API_KEY: null - - MMG_URL: null - MMG_API_KEY: null - MMG_INBOUND_SMS_AUTH: null - MMG_INBOUND_SMS_USERNAME: null - - FIRETEXT_API_KEY: null - LOADTESTING_API_KEY: null - FIRETEXT_INBOUND_SMS_AUTH: null - - REDIS_ENABLED: null - REDIS_URL: null - - TEMPLATE_PREVIEW_API_HOST: null - TEMPLATE_PREVIEW_API_KEY: null - - DOCUMENT_DOWNLOAD_API_HOST: null - DOCUMENT_DOWNLOAD_API_KEY: null - -instances: 1 -memory: 1G -disk_quota: 2G - -applications: - - name: notify-api - - - name: notify-api-db-migration - command: sleep infinity - no-route: true - health-check-type: none - instances: 1 - memory: 128M diff --git a/manifest-api-preview.yml b/manifest-api-preview.yml deleted file mode 100644 index 1eb88ef10..000000000 --- a/manifest-api-preview.yml +++ /dev/null @@ -1,10 +0,0 @@ ---- - -inherit: manifest-api-base.yml - -routes: - - route: notify-api-preview.cloudapps.digital - - route: api.notify.works - -instances: 1 -memory: 1G diff --git a/manifest-api-production.yml b/manifest-api-production.yml deleted file mode 100644 index 6dfdbdaf7..000000000 --- a/manifest-api-production.yml +++ /dev/null @@ -1,9 +0,0 @@ ---- - -inherit: manifest-api-base.yml - -routes: - - route: notify-api-production.cloudapps.digital - - route: api.notifications.service.gov.uk -instances: 2 -memory: 1G diff --git a/manifest-api-sandbox.yml b/manifest-api-sandbox.yml deleted file mode 100644 index feecde9ef..000000000 --- a/manifest-api-sandbox.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- - -inherit: manifest-api-base.yml - -routes: - - route: notify-api-sandbox.cloudapps.digital diff --git a/manifest-api-staging.yml b/manifest-api-staging.yml deleted file mode 100644 index ec6602930..000000000 --- a/manifest-api-staging.yml +++ /dev/null @@ -1,9 +0,0 @@ ---- - -inherit: manifest-api-base.yml - -routes: - - route: notify-api-staging.cloudapps.digital - - route: api.staging-notify.works -instances: 2 -memory: 1G diff --git a/manifest-delivery-base.yml b/manifest-delivery-base.yml deleted file mode 100644 index 2e70b1814..000000000 --- a/manifest-delivery-base.yml +++ /dev/null @@ -1,107 +0,0 @@ ---- - -buildpack: python_buildpack -health-check-type: none -no-route: true - -services: - - notify-db - - logit-ssl-syslog-drain - -instances: 1 -memory: 1G - -env: - # Credentials variables - ADMIN_BASE_URL: null - ADMIN_CLIENT_SECRET: null - API_HOST_NAME: null - DANGEROUS_SALT: null - SECRET_KEY: null - ROUTE_SECRET_KEY_1: null - ROUTE_SECRET_KEY_2: null - CRONITOR_KEYS: null - - PERFORMANCE_PLATFORM_ENDPOINTS: null - - NOTIFICATION_QUEUE_PREFIX: null - AWS_ACCESS_KEY_ID: null - AWS_SECRET_ACCESS_KEY: null - - STATSD_PREFIX: null - - ZENDESK_API_KEY: null - - MMG_URL: null - MMG_API_KEY: null - MMG_INBOUND_SMS_AUTH: null - MMG_INBOUND_SMS_USERNAME: null - - FIRETEXT_API_KEY: null - LOADTESTING_API_KEY: null - FIRETEXT_INBOUND_SMS_AUTH: null - - REDIS_ENABLED: null - REDIS_URL: null - - TEMPLATE_PREVIEW_API_HOST: null - TEMPLATE_PREVIEW_API_KEY: null - SQLALCHEMY_POOL_SIZE: 1 - -applications: - - name: notify-delivery-celery-beat - command: scripts/run_app_paas.sh celery -A run_celery.notify_celery beat --loglevel=INFO - instances: 1 - memory: 128M - env: - NOTIFY_APP_NAME: delivery-celery-beat - - - name: notify-delivery-worker-database - command: scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=11 -Q database-tasks 2> /dev/null - env: - NOTIFY_APP_NAME: delivery-worker-database - - - name: notify-delivery-worker-research - command: scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=5 -Q research-mode-tasks 2> /dev/null - env: - NOTIFY_APP_NAME: delivery-worker-research - - - name: notify-delivery-worker-sender - command: scripts/run_multi_worker_app_paas.sh celery multi start 3 -c 10 -A run_celery.notify_celery --loglevel=INFO -Q send-sms-tasks,send-email-tasks - memory: 3G - disk_quota: 2G - env: - NOTIFY_APP_NAME: delivery-worker-sender - - - name: notify-delivery-worker-periodic - command: scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=2 -Q periodic-tasks,statistics-tasks 2> /dev/null - instances: 1 - env: - NOTIFY_APP_NAME: delivery-worker-periodic - - - name: notify-delivery-worker-priority - command: scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=5 -Q priority-tasks 2> /dev/null - env: - NOTIFY_APP_NAME: delivery-worker-priority - - - name: notify-delivery-worker - command: scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=11 -Q job-tasks,retry-tasks,create-letters-pdf-tasks,letter-tasks 2> /dev/null - env: - NOTIFY_APP_NAME: delivery-worker - - # Only consume the notify-internal-tasks queue on this app so that Notify messages are processed as a priority - - name: notify-delivery-worker-internal - command: scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=11 -Q notify-internal-tasks 2> /dev/null - env: - NOTIFY_APP_NAME: delivery-worker-internal - - - name: notify-delivery-worker-receipts - command: scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=11 -Q ses-callbacks 2> /dev/null - env: - NOTIFY_APP_NAME: delivery-worker-receipts - - - name: notify-delivery-worker-service-callbacks - command: scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=11 -Q service-callbacks 2> /dev/null - disk_quota: 2G - env: - NOTIFY_APP_NAME: delivery-worker-service-callbacks diff --git a/manifest-delivery-preview.yml b/manifest-delivery-preview.yml deleted file mode 100644 index 77fd211c2..000000000 --- a/manifest-delivery-preview.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- - -inherit: manifest-delivery-base.yml - -memory: 1G diff --git a/manifest-delivery-production.yml b/manifest-delivery-production.yml deleted file mode 100644 index 53c8d2f12..000000000 --- a/manifest-delivery-production.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- - -inherit: manifest-delivery-base.yml - -instances: 2 -memory: 1G diff --git a/manifest-delivery-sandbox.yml b/manifest-delivery-sandbox.yml deleted file mode 100644 index d628e5fc9..000000000 --- a/manifest-delivery-sandbox.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- - -inherit: manifest-delivery-base.yml diff --git a/manifest-delivery-staging.yml b/manifest-delivery-staging.yml deleted file mode 100644 index 53c8d2f12..000000000 --- a/manifest-delivery-staging.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- - -inherit: manifest-delivery-base.yml - -instances: 2 -memory: 1G diff --git a/scripts/generate_manifest.py b/scripts/generate_manifest.py deleted file mode 100755 index 8814049c8..000000000 --- a/scripts/generate_manifest.py +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import os -import sys - -import json -import yaml - - -def merge_dicts(a, b): - if not (isinstance(a, dict) and isinstance(b, dict)): - raise ValueError("Error merging variables: '{}' and '{}'".format( - type(a).__name__, type(b).__name__ - )) - - result = a.copy() - for key, val in b.items(): - if isinstance(result.get(key), dict): - result[key] = merge_dicts(a[key], b[key]) - else: - result[key] = val - - return result - - -def load_manifest(manifest_file): - with open(manifest_file) as f: - manifest = yaml.load(f) - - if 'inherit' in manifest: - inherit_file = os.path.join(os.path.dirname(manifest_file), manifest.pop('inherit')) - manifest = merge_dicts(load_manifest(inherit_file), manifest) - - return manifest - - -def load_variables(vars_files): - variables = {} - for vars_file in vars_files: - with open(vars_file) as f: - variables = merge_dicts(variables, yaml.load(f)) - - return { - k.upper(): json.dumps(v) if isinstance(v, (dict, list)) else v - for k, v in variables.items() - } - - -def paas_manifest(manifest_file, *vars_files): - """Generate a PaaS manifest file from a Jinja2 template""" - - manifest = load_manifest(manifest_file) - variables = load_variables(vars_files) - - for key in manifest.get('env', {}): - if key in variables: - manifest['env'][key] = variables[key] - - return yaml.dump(manifest, default_flow_style=False, allow_unicode=True) - - -if __name__ == "__main__": - print('---') - print(paas_manifest(*sys.argv[1:]))