darkhelm/notifications-admin
1
0
Fork 0
mirror of https://github.com/GSA/notifications-admin.git synced 2026-02-26 05:11:06 -05:00
Code Issues Packages Projects Releases Wiki Activity
Files
bde696cf56d86fc78ecb2b80df7e710d730ec24b
notifications-admin/app/config.py
Chris Hill-Scott 24dbe7b7b1 Add Redis cache between admin and API 
Most of the time spent by the admin app to generate a page is spent
waiting for the API. This is slow for three reasons:

1. Talking to the API means going out to the internet, then through
   nginx, the Flask app, SQLAlchemy, down to the database, and then
   serialising the result to JSON and making it into a HTTP response
2. Each call to the API is synchronous, therefore if a page needs 3 API
   calls to render then the second API call won’t be made until the
   first has finished, and the third won’t start until the second has
   finished
3. Every request for a service page in the admin app makes a minimum
   of two requests to the API (`GET /service/…` and `GET /user/…`)

Hitting the database will always be the slowest part of an app like
Notify. But this slowness is exacerbated by 2. and 3. Conversely every
speedup made to 1. is multiplied by 2. and 3.

So this pull request aims to make 1. a _lot_ faster by taking nginx,
Flask, SQLAlchemy and the database out of the equation. It replaces them
with Redis, which as an in-memory key/value store is a lot faster than
Postgres. There is still the overhead of going across the network to
talk to Redis, but the net improvement is vast.

This commit only caches the `GET /service` response, but is written in
such a way that we can easily expand to caching other responses down the
line.

The tradeoff here is that our code is more complex, and we risk
introducing edge cases where a cache becomes stale. The mitigations
against this are:
- invalidating all caches after 24h so a stale cache doesn’t remain
  around indefinitely
- being careful when we add new stuff to the service response

---

Some indicative numbers, based on:
- `GET http://localhost:6012/services/<service_id>/template/<template_id>`
- with the admin app running locally
- talking to Redis running locally
- also talking to the API running locally, itself talking to a local
  Postgres instance
- times measured with Chrome web inspector, average of 10 requests

╲ | No cache | Cache service | Cache service and user | Cache service, user and template
-- | -- | -- | -- | --
**Request time** | 136ms | 97ms | 73ms | 37ms
**Improvement** | 0% | 41% | 88% | 265%

---

Estimates of how much storage this requires:

- Services: 1,942 on production × 2kb = 4Mb
- Users: 4,534 on production × 2kb = 9Mb
- Templates: 7,079 on production × 4kb = 28Mb
2018-04-10 12:58:35 +01:00

170 lines
5.5 KiB
Python
Raw Blame History

import os
if os.environ.get('VCAP_APPLICATION'):
# on cloudfoundry, config is a json blob in VCAP_APPLICATION - unpack it, and populate
# standard environment variables from it
from app.cloudfoundry_config import extract_cloudfoundry_config
extract_cloudfoundry_config()
class Config(object):
ADMIN_CLIENT_SECRET = os.environ.get('ADMIN_CLIENT_SECRET')
API_HOST_NAME = os.environ.get('API_HOST_NAME')
SECRET_KEY = os.environ.get('SECRET_KEY')
DANGEROUS_SALT = os.environ.get('DANGEROUS_SALT')
DESKPRO_API_HOST = os.environ.get('DESKPRO_API_HOST')
DESKPRO_API_KEY = os.environ.get('DESKPRO_API_KEY')
# if we're not on cloudfoundry, we can get to this app from localhost. but on cloudfoundry its different
ADMIN_BASE_URL = os.environ.get('ADMIN_BASE_URL', 'http://localhost:6012')
TEMPLATE_PREVIEW_API_HOST = os.environ.get('TEMPLATE_PREVIEW_API_HOST', 'http://localhost:6013')
TEMPLATE_PREVIEW_API_KEY = os.environ.get('TEMPLATE_PREVIEW_API_KEY', 'my-secret-key')
# Hosted graphite statsd prefix
STATSD_PREFIX = os.getenv('STATSD_PREFIX')
# Logging
DEBUG = False
NOTIFY_LOG_PATH = os.getenv('NOTIFY_LOG_PATH')
DESKPRO_DEPT_ID = 5
DESKPRO_ASSIGNED_AGENT_TEAM_ID = 5
ADMIN_CLIENT_USER_NAME = 'notify-admin'
ASSETS_DEBUG = False
AWS_REGION = 'eu-west-1'
DEFAULT_SERVICE_LIMIT = 50
DEFAULT_FREE_SMS_FRAGMENT_LIMITS = {
'central': 250000,
'local': 25000,
'nhs': 25000,
}
EMAIL_EXPIRY_SECONDS = 3600 # 1 hour
INVITATION_EXPIRY_SECONDS = 3600 * 24 * 2 # 2 days - also set on api
EMAIL_2FA_EXPIRY_SECONDS = 1800 # 30 Minutes
HEADER_COLOUR = '#FFBF47' # $yellow
HTTP_PROTOCOL = 'http'
MAX_FAILED_LOGIN_COUNT = 10
NOTIFY_APP_NAME = 'admin'
NOTIFY_LOG_LEVEL = 'DEBUG'
PERMANENT_SESSION_LIFETIME = 20 * 60 * 60 # 20 hours
SEND_FILE_MAX_AGE_DEFAULT = 365 * 24 * 60 * 60 # 1 year
SESSION_COOKIE_HTTPONLY = True
SESSION_COOKIE_NAME = 'notify_admin_session'
SESSION_COOKIE_SECURE = True
SESSION_REFRESH_EACH_REQUEST = True
SHOW_STYLEGUIDE = True
# TODO: move to utils
SMS_CHAR_COUNT_LIMIT = 459
WTF_CSRF_ENABLED = True
WTF_CSRF_TIME_LIMIT = None
CSV_UPLOAD_BUCKET_NAME = 'local-notifications-csv-upload'
DESKPRO_PERSON_EMAIL = 'donotreply@notifications.service.gov.uk'
ACTIVITY_STATS_LIMIT_DAYS = 7
TEST_MESSAGE_FILENAME = 'Report'
STATSD_ENABLED = False
STATSD_HOST = "statsd.hostedgraphite.com"
STATSD_PORT = 8125
NOTIFY_ENVIRONMENT = 'development'
LOGO_UPLOAD_BUCKET_NAME = 'public-logos-local'
MOU_BUCKET_NAME = 'local-mou'
ROUTE_SECRET_KEY_1 = os.environ.get('ROUTE_SECRET_KEY_1', '')
ROUTE_SECRET_KEY_2 = os.environ.get('ROUTE_SECRET_KEY_2', '')
CHECK_PROXY_HEADER = False
REDIS_URL = os.environ.get('REDIS_URL')
REDIS_ENABLED = os.environ.get('REDIS_ENABLED') == '1'
class Development(Config):
NOTIFY_LOG_PATH = 'application.log'
DEBUG = True
SESSION_COOKIE_SECURE = False
SESSION_PROTECTION = None
STATSD_ENABLED = False
CSV_UPLOAD_BUCKET_NAME = 'development-notifications-csv-upload'
LOGO_UPLOAD_BUCKET_NAME = 'public-logos-tools'
MOU_BUCKET_NAME = 'notify.tools-mou'
ADMIN_CLIENT_SECRET = 'dev-notify-secret-key'
API_HOST_NAME = 'http://localhost:6011'
DANGEROUS_SALT = 'dev-notify-salt'
SECRET_KEY = 'dev-notify-secret-key'
DESKPRO_API_HOST = "some-host"
DESKPRO_API_KEY = "some-key"
class Test(Development):
DEBUG = True
TESTING = True
STATSD_ENABLED = False
WTF_CSRF_ENABLED = False
CSV_UPLOAD_BUCKET_NAME = 'test-notifications-csv-upload'
LOGO_UPLOAD_BUCKET_NAME = 'public-logos-test'
MOU_BUCKET_NAME = 'test-mou'
NOTIFY_ENVIRONMENT = 'test'
API_HOST_NAME = 'http://you-forgot-to-mock-an-api-call-to'
TEMPLATE_PREVIEW_API_HOST = 'http://localhost:9999'
class Preview(Config):
HTTP_PROTOCOL = 'https'
HEADER_COLOUR = '#F499BE' # $baby-pink
STATSD_ENABLED = True
CSV_UPLOAD_BUCKET_NAME = 'preview-notifications-csv-upload'
LOGO_UPLOAD_BUCKET_NAME = 'public-logos-preview'
MOU_BUCKET_NAME = 'notify.works-mou'
NOTIFY_ENVIRONMENT = 'preview'
CHECK_PROXY_HEADER = True
class Staging(Config):
SHOW_STYLEGUIDE = False
HTTP_PROTOCOL = 'https'
HEADER_COLOUR = '#6F72AF' # $mauve
STATSD_ENABLED = True
CSV_UPLOAD_BUCKET_NAME = 'staging-notify-csv-upload'
LOGO_UPLOAD_BUCKET_NAME = 'public-logos-staging'
MOU_BUCKET_NAME = 'staging-notify.works-mou'
NOTIFY_ENVIRONMENT = 'staging'
CHECK_PROXY_HEADER = True
class Live(Config):
SHOW_STYLEGUIDE = False
HEADER_COLOUR = '#005EA5' # $govuk-blue
HTTP_PROTOCOL = 'https'
STATSD_ENABLED = True
CSV_UPLOAD_BUCKET_NAME = 'live-notifications-csv-upload'
LOGO_UPLOAD_BUCKET_NAME = 'public-logos-production'
MOU_BUCKET_NAME = 'notifications.service.gov.uk-mou'
NOTIFY_ENVIRONMENT = 'live'
CHECK_PROXY_HEADER = False
class CloudFoundryConfig(Config):
pass
# CloudFoundry sandbox
class Sandbox(CloudFoundryConfig):
HTTP_PROTOCOL = 'https'
HEADER_COLOUR = '#F499BE' # $baby-pink
STATSD_ENABLED = True
CSV_UPLOAD_BUCKET_NAME = 'cf-sandbox-notifications-csv-upload'
LOGO_UPLOAD_BUCKET_NAME = 'cf-sandbox-notifications-logo-upload'
NOTIFY_ENVIRONMENT = 'sandbox'
configs = {
'development': Development,
'test': Test,
'preview': Preview,
'staging': Staging,
'live': Live,
'production': Live,
'sandbox': Sandbox
}
Reference in New Issue View Git Blame Copy Permalink