Merge pull request #127 from GSA/stdout-logging

Use only stdout logging in cloud.gov
2026-05-14 21:18:41 -04:00 · 2022-11-29 14:26:06 -05:00
parent 8ae134f31c fa8707f802
commit 01a432c86e
10 changed files with 13 additions and 139 deletions
--- a/4
+++ b/4
@@ -49,15 +49,13 @@ lxml = "==4.9.1"
 marshmallow = "==3.15.0"
 marshmallow-sqlalchemy = "==0.28.1"
 notifications-python-client = "==6.3.0"
-notifications-utils = {editable = true, ref = "987646ec18d32dfe8b4ea55b83d68e31e2f85104", git = "https://github.com/GSA/notifications-utils"}
+notifications-utils = {editable = true, ref = "11b9d3dbde8d73910c05b0cb8e74d9f93b45a7b0", git = "https://github.com/GSA/notifications-utils"}
 oscrypto = "==1.3.0"
 psycopg2-binary = "==2.9.3"
 pyjwt = "==2.4.0"
 python-dotenv = "==0.20.0"
 sqlalchemy = "==1.4.40"
 werkzeug = "~=2.1.1"
-# PaaS packages
-awscli-cwlogs = "==1.4.6"
 # gds metrics packages
 prometheus-client = "==0.14.1"
 gds-metrics = {version = "==0.2.4", ref = "6f1840a57b6fb1ee40b7e84f2f18ec229de8aa72", git = "https://github.com/alphagov/gds_metrics_python.git"}
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
    "_meta": {
        "hash": {
-            "sha256": "4dbf825ea628237cbd96a33bf72c1c3a2c82e8a23bd0d53b836667fb8aba040c"
+            "sha256": "adeddc01285ee3e46e1a1464cceffc5fd1d151c61f069642e1c716b8e9fd0507"
        },
        "pipfile-spec": 6,
        "requires": {
@@ -72,13 +72,6 @@
            "index": "pypi",
            "version": "==1.24.8"
        },
-        "awscli-cwlogs": {
-            "hashes": [
-                "sha256:44d2fe77d109b7b630fb8f6c06760ff6c5ec9861be16413fd5b977f5a4971f83"
-            ],
-            "index": "pypi",
-            "version": "==1.4.6"
-        },
        "bcrypt": {
            "hashes": [
                "sha256:2b02d6bfc6336d1094276f3f588aa1225a598e27f8e3388f4db9948cb707b521",
@@ -717,7 +710,7 @@
        "notifications-utils": {
            "editable": true,
            "git": "https://github.com/GSA/notifications-utils",
-            "ref": "987646ec18d32dfe8b4ea55b83d68e31e2f85104"
+            "ref": "11b9d3dbde8d73910c05b0cb8e74d9f93b45a7b0"
        },
        "orderedset": {
            "hashes": [
@@ -1010,10 +1003,10 @@
        },
        "redis": {
            "hashes": [
-                "sha256:a52d5694c9eb4292770084fa8c863f79367ca19884b329ab574d5cb2036b3e54",
-                "sha256:ddf27071df4adf3821c4f2ca59d67525c3a82e5f268bed97b813cb4fabf87880"
+                "sha256:30c07511627a4c5c4d970e060000772f323174f75e745a26938319817ead7a12",
+                "sha256:46652271dc7525cd5a9667e5b0ca983c848c75b2b8f7425403395bb8379dcf25"
            ],
-            "version": "==4.3.4"
+            "version": "==4.3.5"
        },
        "requests": {
            "hashes": [
@@ -1818,11 +1811,11 @@
        },
        "pip-audit": {
            "hashes": [
-                "sha256:6c7fd7c300559c99963f0b4be4543c55175e1550534228b8d27bce7dcd06ab34",
-                "sha256:be866655a365714c98782e00e64b092383ffb9171680aaaa0fb50ba75d775022"
+                "sha256:00ebef2a52884627f255b879135e28001de4378b8005318b66cc3a802459ee0a",
+                "sha256:d6d830bdbe3fd3efaf54f4a203451f286e75aecb7e44f9f84f7bfbd38aba26ac"
            ],
            "index": "pypi",
-            "version": "==2.4.5"
+            "version": "==2.4.6"
        },
        "pip-requirements-parser": {
            "hashes": [
--- a/app/init.py
+++ b/app/init.py
@@ -92,7 +92,7 @@ def create_app(application):
    ma.init_app(application)
    zendesk_client.init_app(application)
    statsd_client.init_app(application)
-    logging.init_app(application, statsd_client)
+    logging.init_app(application)
    aws_sns_client.init_app(application, statsd_client=statsd_client)

    aws_ses_client.init_app(application.config['AWS_REGION'], statsd_client=statsd_client)
--- a/app/config.py
+++ b/app/config.py
@@ -135,7 +135,6 @@ class Config(object):

    # Logging
    DEBUG = False
-    NOTIFY_LOG_PATH = getenv('NOTIFY_LOG_PATH', 'logs/application.log')

    # Monitoring
    CRONITOR_ENABLED = False
--- a/deploy-config/egress_proxy/notify-api-demo.allow.acl
+++ b/deploy-config/egress_proxy/notify-api-demo.allow.acl
@@ -1,3 +1,2 @@
 email.us-west-2.amazonaws.com
 sns.us-west-2.amazonaws.com
-logs.us-west-2.amazonaws.com
--- a/deploy-config/egress_proxy/notify-api-staging.allow.acl
+++ b/deploy-config/egress_proxy/notify-api-staging.allow.acl
@@ -1,3 +1,2 @@
 email.us-west-2.amazonaws.com
 sns.us-west-2.amazonaws.com
-logs.us-west-2.amazonaws.com
--- a/gunicorn_config.py
+++ b/gunicorn_config.py
@@ -10,7 +10,6 @@ from gds_metrics.gunicorn import child_exit  # noqa
 workers = 4
 worker_class = "eventlet"
 worker_connections = 256
-errorlog = "/home/vcap/logs/gunicorn_error.log"
 bind = "0.0.0.0:{}".format(os.getenv("PORT"))
 statsd_host = "{}:8125".format(os.getenv("STATSD_HOST"))
 gunicorn.SERVER_SOFTWARE = 'None'
--- a/manifest.yml
+++ b/manifest.yml
@@ -22,15 +22,14 @@ applications:
      - type: worker
        instances: ((worker_instances))
        memory: ((worker_memory))
-        command: ./scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=4
+        command: celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=4
      - type: scheduler
        instances: 1
        memory: ((scheduler_memory))
-        command: ./scripts/run_app_paas.sh celery -A run_celery.notify_celery beat --loglevel=INFO
+        command: celery -A run_celery.notify_celery beat --loglevel=INFO

    env:
      NOTIFY_APP_NAME: api
-      NOTIFY_LOG_PATH: /home/vcap/logs/app.log
      FLASK_APP: application.py
      FLASK_ENV: production

--- a/scripts/migrate_and_run_web.sh
+++ b/scripts/migrate_and_run_web.sh
@@ -4,4 +4,4 @@ if [[ $CF_INSTANCE_INDEX -eq 0 ]]; then
  flask db upgrade
 fi

-${HOME}/scripts/run_app_paas.sh gunicorn -c ${HOME}/gunicorn_config.py application
+gunicorn -c ${HOME}/gunicorn_config.py application
--- a/scripts/run_app_paas.sh
+++ b/scripts/run_app_paas.sh
@@ -1,112 +0,0 @@
-#!/bin/bash
-
-set -e -o pipefail
-
-TERMINATE_TIMEOUT=9
-MAX_DISK_SPACE_USAGE=75
-readonly LOGS_DIR="/home/vcap/logs"
-
-function check_params {
-  if [ -z "${NOTIFY_APP_NAME}" ]; then
-    echo "You must set NOTIFY_APP_NAME"
-    exit 1
-  fi
-
-  if [ -z "${CW_APP_NAME}" ]; then
-    CW_APP_NAME=${NOTIFY_APP_NAME}
-  fi
-}
-
-function configure_aws_logs {
-  # create files so that aws logs agent doesn't complain
-  touch ${LOGS_DIR}/gunicorn_error.log
-  touch ${LOGS_DIR}/app.log.json
-
-  aws configure set plugins.cwlogs cwlogs
-
-  cat > /home/vcap/app/awslogs.conf << EOF
-[general]
-state_file = ${LOGS_DIR}/awslogs-state
-
-[${LOGS_DIR}/app.log]
-file = ${LOGS_DIR}/app.log.json
-log_group_name = paas-${CW_APP_NAME}-application
-log_stream_name = {hostname}
-
-[${LOGS_DIR}/gunicorn_error.log]
-file = ${LOGS_DIR}/gunicorn_error.log
-log_group_name = paas-${CW_APP_NAME}-gunicorn
-log_stream_name = {hostname}
-EOF
-}
-
-function on_exit {
-  echo "Terminating application process with pid ${APP_PID}"
-  kill ${APP_PID} || true
-  wait_time=0
-  while (kill -0 ${APP_PID} 2&>/dev/null); do
-    echo "Application is still running.."
-    sleep 1
-    let wait_time=wait_time+1
-    if [ "$wait_time" -ge "$TERMINATE_TIMEOUT" ]; then
-      echo "Timeout reached, killing process with pid ${APP_PID}"
-      kill -9 ${APP_PID} || true
-      break
-    fi
-  done
-  echo "Terminating remaining subprocesses.."
-  kill 0
-}
-
-function start_application {
-  exec "$@" &
-  APP_PID=`jobs -p`
-  echo "Application process pid: ${APP_PID}"
-}
-
-function start_aws_logs_agent {
-  exec aws logs push --region us-west-2 --config-file /home/vcap/app/awslogs.conf &
-  AWSLOGS_AGENT_PID=$!
-  echo "AWS logs agent pid: ${AWSLOGS_AGENT_PID}"
-}
-
-function check_disk_space {
-    # get something like:
-    #
-    # Filesystem     Use%
-    # overlay         56%
-    # tmpfs            0%
-    #
-    # and only keep '56'
-    SPACE_USAGE=$(df --output="source,pcent" | grep overlay | tr --squeeze-repeats " " | cut -f2 -d" "| cut -f1 -d"%")
-
-    if [[ "${SPACE_USAGE}" -ge "${MAX_DISK_SPACE_USAGE}" ]]; then
-        echo "Terminating ${NOTIFY_APP_NAME}, instance ${INSTANCE_INDEX} because we're running out of disk space"
-        echo "Usage: ${SPACE_USAGE}% - limit ${MAX_DISK_SPACE_USAGE}%"
-        exit
-    fi
-}
-
-function run {
-  while true; do
-    kill -0 ${APP_PID} 2&>/dev/null || break
-    kill -0 ${AWSLOGS_AGENT_PID} 2&>/dev/null || start_aws_logs_agent
-    check_disk_space
-    sleep 1
-  done
-}
-
-echo "Run script pid: $$"
-
-check_params
-
-trap "on_exit" EXIT
-
-configure_aws_logs
-
-# The application has to start first!
-start_application "$@"
-
-start_aws_logs_agent
-
-run