From 213e32cf6d9d0703a6e10edc65cb8c17608947f0 Mon Sep 17 00:00:00 2001 From: Rebecca Law Date: Thu, 28 Jun 2018 17:09:45 +0100 Subject: [PATCH 1/5] Added a one off command to add many invited user to a service. It would be nice to add something that can do this from the Admin app. --- app/commands.py | 45 +++++++++++++++++++++++++++++++++++++++++++++ app/invite/rest.py | 4 ++++ 2 files changed, 49 insertions(+) diff --git a/app/commands.py b/app/commands.py index eda6ae69e..e3f1a9923 100644 --- a/app/commands.py +++ b/app/commands.py @@ -33,6 +33,7 @@ from app.dao.services_dao import ( dao_fetch_service_by_id ) from app.dao.users_dao import (delete_model_user, delete_user_verify_codes) +from app.invite.rest import create_invited_user from app.models import PROVIDERS, User, SMS_TYPE, EMAIL_TYPE, Notification from app.performance_platform.processing_time import (send_processing_time_for_start_and_end) from app.utils import ( @@ -622,3 +623,47 @@ def migrate_data_to_ft_notification_status(start_date, end_date): total_updated += result.rowcount print('Total inserted/updated records = {}'.format(total_updated)) + + +@notify_command(name='bulk-invite-user-to-service') +@click.option('-f', '--file_name', required=True, + help="Full path of the file containing a list of email address for people to invite to a service") +@click.option('-s', '--service_id', required=True, help='The id of the service that the invite is for') +@click.option('-u', '--user_id', required=True, help='The id of the user that the invite is from') +@click.option('-a', '--auth_type', required=False, + help='The authentication type for the user, sms_auth or email_auth. Defaults to sms_auth if not provided') +@click.option('-p', '--permissions', required=True, help='Comma separated list of permissions.') +def bulk_invite_user_to_service(file_name, service_id, user_id, auth_type, permissions): + # permissions + # manage_users | manage_templates | manage_settings + # send messages ==> send_texts | send_emails | send_letters + # Access API keys manage_api_keys + # platform_admin + # view_activity + # "send_texts,send_emails,send_letters,view_activity" + + file = open(file_name) + for email_address in file: + data = { + 'service': service_id, + 'email_address': email_address.strip(), + 'from_user': user_id, + 'permissions': permissions, + 'auth_type': auth_type, + 'invite_link_host': current_app.config['ADMIN_BASE_URL'] + } + with current_app.test_request_context( + path='/service/{}/invite/'.format(service_id), + method='POST', + data=json.dumps(data), + headers={"Content-Type": "application/json"} + ): + try: + response = create_invited_user(service_id) + if response[1] != 201: + print("*** ERROR occurred for email address: {}".format(email_address.strip())) + print(response[0].get_data(as_text=True)) + except Exception as e: + print("*** ERROR occurred for email address: {}. \n{}".format(email_address.strip(), e)) + + file.close() diff --git a/app/invite/rest.py b/app/invite/rest.py index fcd7e7b08..0d6c86bb3 100644 --- a/app/invite/rest.py +++ b/app/invite/rest.py @@ -1,3 +1,5 @@ +import json + from flask import ( Blueprint, request, @@ -24,6 +26,8 @@ register_errors(invite) @invite.route('', methods=['POST']) def create_invited_user(service_id): request_json = request.get_json() + # request_json = request.data + print(request_json) invited_user, errors = invited_user_schema.load(request_json) save_invited_user(invited_user) From 1c960272a62cbdcb8a5d6a7ce7440d43416bef63 Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Fri, 29 Jun 2018 10:09:18 +0100 Subject: [PATCH 2/5] Add Ofgem letter logo --- .../versions/0200_another_letter_org.py | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 migrations/versions/0200_another_letter_org.py diff --git a/migrations/versions/0200_another_letter_org.py b/migrations/versions/0200_another_letter_org.py new file mode 100644 index 000000000..ed5ce48f0 --- /dev/null +++ b/migrations/versions/0200_another_letter_org.py @@ -0,0 +1,36 @@ +"""empty message + +Revision ID: 0200_another_letter_org +Revises: 0199_another_letter_org +Create Date: 2017-06-29 12:44:16.815039 + +""" + +# revision identifiers, used by Alembic. +revision = '0200_another_letter_org' +down_revision = '0199_another_letter_org' + +from alembic import op + + +NEW_ORGANISATIONS = [ + ('508', 'Ofgem'), +] + + +def upgrade(): + for numeric_id, name in NEW_ORGANISATIONS: + op.execute(""" + INSERT + INTO dvla_organisation + VALUES ('{}', '{}') + """.format(numeric_id, name)) + + +def downgrade(): + for numeric_id, _ in NEW_ORGANISATIONS: + op.execute(""" + DELETE + FROM dvla_organisation + WHERE id = '{}' + """.format(numeric_id)) From b201711705d292a157f92de2bd460e9eb4cd76b7 Mon Sep 17 00:00:00 2001 From: Rebecca Law Date: Fri, 29 Jun 2018 10:26:14 +0100 Subject: [PATCH 3/5] Undo the chnages to create_invited_user --- app/invite/rest.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/app/invite/rest.py b/app/invite/rest.py index 0d6c86bb3..fcd7e7b08 100644 --- a/app/invite/rest.py +++ b/app/invite/rest.py @@ -1,5 +1,3 @@ -import json - from flask import ( Blueprint, request, @@ -26,8 +24,6 @@ register_errors(invite) @invite.route('', methods=['POST']) def create_invited_user(service_id): request_json = request.get_json() - # request_json = request.data - print(request_json) invited_user, errors = invited_user_schema.load(request_json) save_invited_user(invited_user) From f4846c88c9f281daf43dc4246bf3ea10c44e9dcc Mon Sep 17 00:00:00 2001 From: Rebecca Law Date: Fri, 29 Jun 2018 11:12:54 +0100 Subject: [PATCH 4/5] Had to move the import to the method rather than the top of the file. --- app/commands.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/app/commands.py b/app/commands.py index e3f1a9923..b130d97a8 100644 --- a/app/commands.py +++ b/app/commands.py @@ -33,7 +33,6 @@ from app.dao.services_dao import ( dao_fetch_service_by_id ) from app.dao.users_dao import (delete_model_user, delete_user_verify_codes) -from app.invite.rest import create_invited_user from app.models import PROVIDERS, User, SMS_TYPE, EMAIL_TYPE, Notification from app.performance_platform.processing_time import (send_processing_time_for_start_and_end) from app.utils import ( @@ -641,7 +640,7 @@ def bulk_invite_user_to_service(file_name, service_id, user_id, auth_type, permi # platform_admin # view_activity # "send_texts,send_emails,send_letters,view_activity" - + from app.invite.rest import create_invited_user file = open(file_name) for email_address in file: data = { From 676e3ec39aabeb6667864eea06186ff29ff069c5 Mon Sep 17 00:00:00 2001 From: Alexey Bezhan Date: Fri, 29 Jun 2018 11:49:02 +0100 Subject: [PATCH 5/5] Stream delivery worker logs to stdout when running on PaaS Our application servers and celery workers write logs both to a file that is shipped to CloudWatch and to stdout, which is picked up by CloudFoundry and sent to Logit Logstash. This works with gunicorn and single-worker celery deployments, however celery multi daemonizes worker processes, which detaches them from stdout, so there's no log output in `cf logs` or Logit. To fix this, we start a separate tail process to duplicate logs written to a file to stdout, which should be picked up by CloudFoundry. --- scripts/run_multi_worker_app_paas.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/scripts/run_multi_worker_app_paas.sh b/scripts/run_multi_worker_app_paas.sh index d50358c6b..3965ee577 100755 --- a/scripts/run_multi_worker_app_paas.sh +++ b/scripts/run_multi_worker_app_paas.sh @@ -97,6 +97,12 @@ function start_aws_logs_agent { echo "AWS logs agent pid: ${AWSLOGS_AGENT_PID}" } +function start_logs_tail { + exec tail -n0 -f ${LOGS_DIR}/app.log.json & + LOGS_TAIL_PID=$! + echo "tail pid: ${LOGS_TAIL_PID}" +} + function run { while true; do get_celery_pids @@ -104,6 +110,7 @@ function run { kill -0 ${APP_PID} 2&>/dev/null || return 1 done kill -0 ${AWSLOGS_AGENT_PID} 2&>/dev/null || start_aws_logs_agent + kill -0 ${LOGS_TAIL_PID} 2&>/dev/null || start_logs_tail sleep 1 done } @@ -120,5 +127,6 @@ configure_aws_logs start_application "$@" start_aws_logs_agent +start_logs_tail run