mirror of
https://github.com/GSA/notifications-admin.git
synced 2026-02-05 10:53:28 -05:00
617 lines
22 KiB
Python
617 lines
22 KiB
Python
import itertools
|
||
import re
|
||
from collections import OrderedDict
|
||
from datetime import datetime
|
||
|
||
from flask import abort, flash, redirect, render_template, request, url_for
|
||
from notifications_python_client.errors import HTTPError
|
||
|
||
from app import (
|
||
billing_api_client,
|
||
complaint_api_client,
|
||
format_date_numeric,
|
||
letter_jobs_client,
|
||
notification_api_client,
|
||
platform_stats_api_client,
|
||
service_api_client,
|
||
)
|
||
from app.extensions import redis_client
|
||
from app.main import main
|
||
from app.main.forms import (
|
||
AdminClearCacheForm,
|
||
AdminReturnedLettersForm,
|
||
BillingReportDateFilterForm,
|
||
DateFilterForm,
|
||
RequiredDateFilterForm,
|
||
)
|
||
from app.statistics_utils import (
|
||
get_formatted_percentage,
|
||
get_formatted_percentage_two_dp,
|
||
)
|
||
from app.utils.csv import Spreadsheet
|
||
from app.utils.pagination import (
|
||
generate_next_dict,
|
||
generate_previous_dict,
|
||
get_page_from_request,
|
||
)
|
||
from app.utils.user import user_is_platform_admin
|
||
|
||
COMPLAINT_THRESHOLD = 0.02
|
||
FAILURE_THRESHOLD = 3
|
||
ZERO_FAILURE_THRESHOLD = 0
|
||
|
||
|
||
@main.route("/platform-admin")
|
||
@user_is_platform_admin
|
||
def platform_admin_splash_page():
|
||
return render_template(
|
||
'views/platform-admin/splash-page.html',
|
||
)
|
||
|
||
|
||
@main.route("/platform-admin/summary")
|
||
@user_is_platform_admin
|
||
def platform_admin():
|
||
form = DateFilterForm(request.args, meta={'csrf': False})
|
||
api_args = {}
|
||
|
||
form.validate()
|
||
|
||
if form.start_date.data:
|
||
api_args['start_date'] = form.start_date.data
|
||
api_args['end_date'] = form.end_date.data or datetime.utcnow().date()
|
||
|
||
platform_stats = platform_stats_api_client.get_aggregate_platform_stats(api_args)
|
||
number_of_complaints = complaint_api_client.get_complaint_count(api_args)
|
||
|
||
return render_template(
|
||
'views/platform-admin/index.html',
|
||
form=form,
|
||
global_stats=make_columns(platform_stats, number_of_complaints)
|
||
)
|
||
|
||
|
||
def is_over_threshold(number, total, threshold):
|
||
percentage = number / total * 100 if total else 0
|
||
return percentage > threshold
|
||
|
||
|
||
def get_status_box_data(stats, key, label, threshold=FAILURE_THRESHOLD):
|
||
return {
|
||
'number': "{:,}".format(stats['failures'][key]),
|
||
'label': label,
|
||
'failing': is_over_threshold(
|
||
stats['failures'][key],
|
||
stats['total'],
|
||
threshold
|
||
),
|
||
'percentage': get_formatted_percentage(stats['failures'][key], stats['total'])
|
||
}
|
||
|
||
|
||
def get_tech_failure_status_box_data(stats):
|
||
stats = get_status_box_data(stats, 'technical-failure', 'technical failures', ZERO_FAILURE_THRESHOLD)
|
||
stats.pop('percentage')
|
||
return stats
|
||
|
||
|
||
def make_columns(global_stats, complaints_number):
|
||
return [
|
||
# email
|
||
{
|
||
'black_box': {
|
||
'number': global_stats['email']['total'],
|
||
'notification_type': 'email'
|
||
},
|
||
'other_data': [
|
||
get_tech_failure_status_box_data(global_stats['email']),
|
||
get_status_box_data(global_stats['email'], 'permanent-failure', 'permanent failures'),
|
||
get_status_box_data(global_stats['email'], 'temporary-failure', 'temporary failures'),
|
||
{
|
||
'number': complaints_number,
|
||
'label': 'complaints',
|
||
'failing': is_over_threshold(complaints_number,
|
||
global_stats['email']['total'], COMPLAINT_THRESHOLD),
|
||
'percentage': get_formatted_percentage_two_dp(complaints_number, global_stats['email']['total']),
|
||
'url': url_for('main.platform_admin_list_complaints')
|
||
}
|
||
],
|
||
'test_data': {
|
||
'number': global_stats['email']['test-key'],
|
||
'label': 'test emails'
|
||
}
|
||
},
|
||
# sms
|
||
{
|
||
'black_box': {
|
||
'number': global_stats['sms']['total'],
|
||
'notification_type': 'sms'
|
||
},
|
||
'other_data': [
|
||
get_tech_failure_status_box_data(global_stats['sms']),
|
||
get_status_box_data(global_stats['sms'], 'permanent-failure', 'permanent failures'),
|
||
get_status_box_data(global_stats['sms'], 'temporary-failure', 'temporary failures')
|
||
],
|
||
'test_data': {
|
||
'number': global_stats['sms']['test-key'],
|
||
'label': 'test text messages'
|
||
}
|
||
},
|
||
# letter
|
||
{
|
||
'black_box': {
|
||
'number': global_stats['letter']['total'],
|
||
'notification_type': 'letter'
|
||
},
|
||
'other_data': [
|
||
get_tech_failure_status_box_data(global_stats['letter']),
|
||
get_status_box_data(global_stats['letter'],
|
||
'virus-scan-failed', 'virus scan failures', ZERO_FAILURE_THRESHOLD)
|
||
],
|
||
'test_data': {
|
||
'number': global_stats['letter']['test-key'],
|
||
'label': 'test letters'
|
||
}
|
||
},
|
||
]
|
||
|
||
|
||
@main.route("/platform-admin/live-services", endpoint='live_services')
|
||
@main.route("/platform-admin/trial-services", endpoint='trial_services')
|
||
@user_is_platform_admin
|
||
def platform_admin_services():
|
||
form = DateFilterForm(request.args)
|
||
if all((
|
||
request.args.get('include_from_test_key') is None,
|
||
request.args.get('start_date') is None,
|
||
request.args.get('end_date') is None,
|
||
)):
|
||
# Default to True if the user hasn’t done any filtering,
|
||
# otherwise respect their choice
|
||
form.include_from_test_key.data = True
|
||
|
||
include_from_test_key = form.include_from_test_key.data
|
||
api_args = {'detailed': True,
|
||
'only_active': False, # specifically DO get inactive services
|
||
'include_from_test_key': include_from_test_key,
|
||
}
|
||
|
||
if form.start_date.data:
|
||
api_args['start_date'] = form.start_date.data
|
||
api_args['end_date'] = form.end_date.data or datetime.utcnow().date()
|
||
|
||
services = filter_and_sort_services(
|
||
service_api_client.get_services(api_args)['data'],
|
||
trial_mode_services=request.endpoint == 'main.trial_services',
|
||
)
|
||
|
||
return render_template(
|
||
'views/platform-admin/services.html',
|
||
include_from_test_key=include_from_test_key,
|
||
form=form,
|
||
services=list(format_stats_by_service(services)),
|
||
page_title='{} services'.format(
|
||
'Trial mode' if request.endpoint == 'main.trial_services' else 'Live'
|
||
),
|
||
global_stats=create_global_stats(services),
|
||
)
|
||
|
||
|
||
@main.route("/platform-admin/reports")
|
||
@user_is_platform_admin
|
||
def platform_admin_reports():
|
||
return render_template(
|
||
'views/platform-admin/reports.html'
|
||
)
|
||
|
||
|
||
@main.route("/platform-admin/reports/live-services.csv")
|
||
@user_is_platform_admin
|
||
def live_services_csv():
|
||
results = service_api_client.get_live_services_data()["data"]
|
||
|
||
column_names = OrderedDict([
|
||
('service_id', 'Service ID'),
|
||
('organisation_name', 'Organisation'),
|
||
('organisation_type', 'Organisation type'),
|
||
('service_name', 'Service name'),
|
||
('consent_to_research', 'Consent to research'),
|
||
('contact_name', 'Main contact'),
|
||
('contact_email', 'Contact email'),
|
||
('contact_mobile', 'Contact mobile'),
|
||
('live_date', 'Live date'),
|
||
('sms_volume_intent', 'SMS volume intent'),
|
||
('email_volume_intent', 'Email volume intent'),
|
||
('letter_volume_intent', 'Letter volume intent'),
|
||
('sms_totals', 'SMS sent this year'),
|
||
('email_totals', 'Emails sent this year'),
|
||
('letter_totals', 'Letters sent this year'),
|
||
('free_sms_fragment_limit', 'Free sms allowance'),
|
||
])
|
||
|
||
# initialise with header row
|
||
live_services_data = [[x for x in column_names.values()]]
|
||
|
||
for row in results:
|
||
if row['live_date']:
|
||
row['live_date'] = datetime.strptime(row["live_date"], '%a, %d %b %Y %X %Z').strftime("%d-%m-%Y")
|
||
|
||
live_services_data.append([row[api_key] for api_key in column_names.keys()])
|
||
|
||
return Spreadsheet.from_rows(live_services_data).as_csv_data, 200, {
|
||
'Content-Type': 'text/csv; charset=utf-8',
|
||
'Content-Disposition': 'inline; filename="{} live services report.csv"'.format(
|
||
format_date_numeric(datetime.now().strftime("%Y-%m-%dT%H:%M:%S.%fZ")),
|
||
)
|
||
}
|
||
|
||
|
||
@main.route("/platform-admin/reports/notifications-sent-by-service", methods=['GET', 'POST'])
|
||
@user_is_platform_admin
|
||
def notifications_sent_by_service():
|
||
form = RequiredDateFilterForm()
|
||
|
||
if form.validate_on_submit():
|
||
start_date = form.start_date.data
|
||
end_date = form.end_date.data
|
||
|
||
headers = [
|
||
'date_created', 'service_id', 'service_name', 'notification_type', 'count_sending', 'count_delivered',
|
||
'count_technical_failure', 'count_temporary_failure', 'count_permanent_failure', 'count_sent'
|
||
]
|
||
result = notification_api_client.get_notification_status_by_service(start_date, end_date)
|
||
|
||
return Spreadsheet.from_rows([headers] + result).as_csv_data, 200, {
|
||
'Content-Type': 'text/csv; charset=utf-8',
|
||
'Content-Disposition': 'attachment; filename="{} to {} notification status per service report.csv"'.format(
|
||
start_date, end_date)
|
||
}
|
||
|
||
return render_template('views/platform-admin/notifications_by_service.html', form=form)
|
||
|
||
|
||
@main.route("/platform-admin/reports/usage-for-all-services", methods=['GET', 'POST'])
|
||
@user_is_platform_admin
|
||
def get_billing_report():
|
||
form = BillingReportDateFilterForm()
|
||
|
||
if form.validate_on_submit():
|
||
start_date = form.start_date.data
|
||
end_date = form.end_date.data
|
||
headers = [
|
||
"organisation_id", "organisation_name", "service_id", "service_name",
|
||
"sms_cost", "sms_chargeable_units", "total_letters", "letter_cost", "letter_breakdown",
|
||
"purchase_order_number", "contact_names", "contact_email_addresses", "billing_reference"
|
||
]
|
||
try:
|
||
result = billing_api_client.get_data_for_billing_report(start_date, end_date)
|
||
except HTTPError as e:
|
||
message = 'Date must be in a single financial year.'
|
||
if e.status_code == 400 and e.message == message:
|
||
flash(message)
|
||
return render_template('views/platform-admin/get-billing-report.html', form=form)
|
||
else:
|
||
raise e
|
||
rows = [
|
||
[
|
||
r["organisation_id"], r["organisation_name"], r["service_id"], r["service_name"],
|
||
r["sms_cost"], r["sms_chargeable_units"], r["total_letters"], r["letter_cost"],
|
||
r["letter_breakdown"].strip(), r.get("purchase_order_number"), r.get("contact_names"),
|
||
r.get("contact_email_addresses"), r.get("billing_reference")
|
||
]
|
||
for r in result
|
||
]
|
||
if rows:
|
||
return Spreadsheet.from_rows([headers] + rows).as_csv_data, 200, {
|
||
'Content-Type': 'text/csv; charset=utf-8',
|
||
'Content-Disposition': 'attachment; filename="Billing Report from {} to {}.csv"'.format(
|
||
start_date, end_date
|
||
)
|
||
}
|
||
else:
|
||
flash('No results for dates')
|
||
return render_template('views/platform-admin/get-billing-report.html', form=form)
|
||
|
||
|
||
@main.route("/platform-admin/reports/volumes-by-service", methods=['GET', 'POST'])
|
||
@user_is_platform_admin
|
||
def get_volumes_by_service():
|
||
form = BillingReportDateFilterForm()
|
||
|
||
if form.validate_on_submit():
|
||
start_date = form.start_date.data
|
||
end_date = form.end_date.data
|
||
headers = [
|
||
"organisation id", "organisation name", "service id", "service name",
|
||
"free allowance", "sms notifications", "sms chargeable units", "email totals",
|
||
"letter totals", "letter cost", "letter sheet totals"
|
||
]
|
||
result = billing_api_client.get_data_for_volumes_by_service_report(start_date, end_date)
|
||
|
||
rows = [
|
||
[
|
||
r["organisation_id"], r["organisation_name"], r["service_id"], r["service_name"],
|
||
r["free_allowance"], r["sms_notifications"], r["sms_chargeable_units"], r["email_totals"],
|
||
r["letter_totals"], r["letter_cost"], r["letter_sheet_totals"]
|
||
]
|
||
for r in result
|
||
]
|
||
if rows:
|
||
return Spreadsheet.from_rows([headers] + rows).as_csv_data, 200, {
|
||
'Content-Type': 'text/csv; charset=utf-8',
|
||
'Content-Disposition': 'attachment; filename="Volumes by service report from {} to {}.csv"'.format(
|
||
start_date, end_date
|
||
)
|
||
}
|
||
else:
|
||
flash('No results for dates')
|
||
return render_template('views/platform-admin/volumes-by-service-report.html', form=form)
|
||
|
||
|
||
@main.route("/platform-admin/reports/daily-volumes-report", methods=['GET', 'POST'])
|
||
@user_is_platform_admin
|
||
def get_daily_volumes():
|
||
form = BillingReportDateFilterForm()
|
||
|
||
if form.validate_on_submit():
|
||
start_date = form.start_date.data
|
||
end_date = form.end_date.data
|
||
headers = [
|
||
"day", "sms totals", "sms fragment totals", "sms chargeable units",
|
||
"email totals", "letter totals", "letter sheet totals"
|
||
]
|
||
result = billing_api_client.get_data_for_daily_volumes_report(start_date, end_date)
|
||
|
||
rows = [
|
||
[
|
||
r["day"], r["sms_totals"], r["sms_fragment_totals"], r["sms_chargeable_units"],
|
||
r["email_totals"], r["letter_totals"], r["letter_sheet_totals"]
|
||
]
|
||
for r in result
|
||
]
|
||
if rows:
|
||
return Spreadsheet.from_rows([headers] + rows).as_csv_data, 200, {
|
||
'Content-Type': 'text/csv; charset=utf-8',
|
||
'Content-Disposition': 'attachment; filename="Daily volumes report from {} to {}.csv"'.format(
|
||
start_date, end_date
|
||
)
|
||
}
|
||
else:
|
||
flash('No results for dates')
|
||
return render_template('views/platform-admin/daily-volumes-report.html', form=form)
|
||
|
||
|
||
@main.route("/platform-admin/reports/daily-sms-provider-volumes-report", methods=['GET', 'POST'])
|
||
@user_is_platform_admin
|
||
def get_daily_sms_provider_volumes():
|
||
form = BillingReportDateFilterForm()
|
||
|
||
if form.validate_on_submit():
|
||
start_date = form.start_date.data
|
||
end_date = form.end_date.data
|
||
headers = [
|
||
"day",
|
||
"provider",
|
||
"sms totals",
|
||
"sms fragment totals",
|
||
"sms chargeable units",
|
||
"sms cost",
|
||
]
|
||
result = billing_api_client.get_data_for_daily_sms_provider_volumes_report(start_date, end_date)
|
||
|
||
rows = [
|
||
[
|
||
r["day"],
|
||
r["provider"],
|
||
r["sms_totals"],
|
||
r["sms_fragment_totals"],
|
||
r["sms_chargeable_units"],
|
||
r["sms_cost"]
|
||
]
|
||
for r in result
|
||
]
|
||
if rows:
|
||
return Spreadsheet.from_rows([headers] + rows).as_csv_data, 200, {
|
||
'Content-Type': 'text/csv; charset=utf-8',
|
||
'Content-Disposition':
|
||
f'attachment; filename="Daily SMS provider volumes report from {start_date} to {end_date}.csv"'
|
||
}
|
||
else:
|
||
flash('No results for dates')
|
||
return render_template('views/platform-admin/daily-sms-provider-volumes-report.html', form=form)
|
||
|
||
|
||
@main.route("/platform-admin/complaints")
|
||
@user_is_platform_admin
|
||
def platform_admin_list_complaints():
|
||
page = get_page_from_request()
|
||
if page is None:
|
||
abort(404, "Invalid page argument ({}).".format(request.args.get('page')))
|
||
|
||
response = complaint_api_client.get_all_complaints(page=page)
|
||
|
||
prev_page = None
|
||
if response['links'].get('prev'):
|
||
prev_page = generate_previous_dict('main.platform_admin_list_complaints', None, page)
|
||
next_page = None
|
||
if response['links'].get('next'):
|
||
next_page = generate_next_dict('main.platform_admin_list_complaints', None, page)
|
||
|
||
return render_template(
|
||
'views/platform-admin/complaints.html',
|
||
complaints=response['complaints'],
|
||
page=page,
|
||
prev_page=prev_page,
|
||
next_page=next_page,
|
||
)
|
||
|
||
|
||
@main.route("/platform-admin/returned-letters", methods=["GET", "POST"])
|
||
@user_is_platform_admin
|
||
def platform_admin_returned_letters():
|
||
form = AdminReturnedLettersForm()
|
||
|
||
if form.validate_on_submit():
|
||
references = [
|
||
re.sub('NOTIFY00[0-9]', '', r.strip())
|
||
for r in form.references.data.split('\n')
|
||
if r.strip()
|
||
]
|
||
|
||
try:
|
||
letter_jobs_client.submit_returned_letters(references)
|
||
redis_client.delete_by_pattern(
|
||
'service-????????-????-????-????-????????????-returned-letters-statistics'
|
||
)
|
||
redis_client.delete_by_pattern(
|
||
'service-????????-????-????-????-????????????-returned-letters-summary'
|
||
)
|
||
except HTTPError as error:
|
||
if error.status_code == 400:
|
||
error_references = [
|
||
re.match('references (.*) does not match', e['message']).group(1)
|
||
for e in error.message
|
||
]
|
||
form.references.errors.append("Invalid references: {}".format(', '.join(error_references)))
|
||
else:
|
||
raise error
|
||
else:
|
||
flash('Submitted {} letter references'.format(len(references)), 'default')
|
||
return redirect(
|
||
url_for('.platform_admin_returned_letters')
|
||
)
|
||
return render_template(
|
||
'views/platform-admin/returned-letters.html',
|
||
form=form,
|
||
)
|
||
|
||
|
||
@main.route("/platform-admin/clear-cache", methods=['GET', 'POST'])
|
||
@user_is_platform_admin
|
||
def clear_cache():
|
||
# note: `service-{uuid}-templates` cache is cleared for both services and templates.
|
||
CACHE_KEYS = OrderedDict([
|
||
('user', [
|
||
'user-????????-????-????-????-????????????',
|
||
]),
|
||
('service', [
|
||
'has_jobs-????????-????-????-????-????????????',
|
||
'service-????????-????-????-????-????????????',
|
||
'service-????????-????-????-????-????????????-templates',
|
||
'service-????????-????-????-????-????????????-data-retention',
|
||
'service-????????-????-????-????-????????????-template-folders',
|
||
'service-????????-????-????-????-????????????-returned-letters-statistics',
|
||
'service-????????-????-????-????-????????????-returned-letters-summary',
|
||
]),
|
||
('template', [
|
||
'service-????????-????-????-????-????????????-templates',
|
||
'service-????????-????-????-????-????????????-template-????????-????-????-????-????????????-version-*',
|
||
'service-????????-????-????-????-????????????-template-????????-????-????-????-????????????-versions',
|
||
]),
|
||
('email_branding', [
|
||
'email_branding',
|
||
'email_branding-????????-????-????-????-????????????',
|
||
]),
|
||
('letter_branding', [
|
||
'letter_branding',
|
||
'letter_branding-????????-????-????-????-????????????',
|
||
]),
|
||
('organisation', [
|
||
'organisations',
|
||
'domains',
|
||
'live-service-and-organisation-counts',
|
||
'organisation-????????-????-????-????-????????????-name',
|
||
]),
|
||
])
|
||
|
||
form = AdminClearCacheForm()
|
||
|
||
form.model_type.choices = [
|
||
(key, key.replace('_', ' ').title()) for key in CACHE_KEYS
|
||
]
|
||
|
||
if form.validate_on_submit():
|
||
group_keys = form.model_type.data
|
||
groups = map(CACHE_KEYS.get, group_keys)
|
||
patterns = list(itertools.chain(*groups))
|
||
|
||
num_deleted = sum(
|
||
redis_client.delete_by_pattern(pattern)
|
||
for pattern in patterns
|
||
)
|
||
|
||
msg = (
|
||
f'Removed {num_deleted} objects '
|
||
f'across {len(patterns)} key formats '
|
||
f'for {", ".join(group_keys)}'
|
||
)
|
||
|
||
flash(msg, category='default')
|
||
|
||
return render_template(
|
||
'views/platform-admin/clear-cache.html',
|
||
form=form
|
||
)
|
||
|
||
|
||
def sum_service_usage(service):
|
||
total = 0
|
||
for notification_type in service['statistics'].keys():
|
||
total += service['statistics'][notification_type]['requested']
|
||
return total
|
||
|
||
|
||
def filter_and_sort_services(services, trial_mode_services=False):
|
||
return [
|
||
service for service in sorted(
|
||
services,
|
||
key=lambda service: (
|
||
service['active'],
|
||
sum_service_usage(service),
|
||
service['created_at']
|
||
),
|
||
reverse=True,
|
||
)
|
||
if service['restricted'] == trial_mode_services
|
||
]
|
||
|
||
|
||
def create_global_stats(services):
|
||
stats = {
|
||
'email': {
|
||
'delivered': 0,
|
||
'failed': 0,
|
||
'requested': 0
|
||
},
|
||
'sms': {
|
||
'delivered': 0,
|
||
'failed': 0,
|
||
'requested': 0
|
||
},
|
||
'letter': {
|
||
'delivered': 0,
|
||
'failed': 0,
|
||
'requested': 0
|
||
}
|
||
}
|
||
for service in services:
|
||
for msg_type, status in itertools.product(('sms', 'email', 'letter'), ('delivered', 'failed', 'requested')):
|
||
stats[msg_type][status] += service['statistics'][msg_type][status]
|
||
|
||
for stat in stats.values():
|
||
stat['failure_rate'] = get_formatted_percentage(stat['failed'], stat['requested'])
|
||
return stats
|
||
|
||
|
||
def format_stats_by_service(services):
|
||
for service in services:
|
||
yield {
|
||
'id': service['id'],
|
||
'name': service['name'],
|
||
'stats': service['statistics'],
|
||
'restricted': service['restricted'],
|
||
'research_mode': service['research_mode'],
|
||
'created_at': service['created_at'],
|
||
'active': service['active']
|
||
}
|