From 7a95e1618e760a49235ddf058956f6fc7cbd3610 Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Wed, 6 Jan 2021 12:12:01 +0000 Subject: [PATCH] Extract formatters into their own module We have lots of functions for converting various types of data into strings to be displayed to the user somewhere. This commit collects all these functions into their own module, rather than having them cluttering up `app/__init__.py` or buried amongst various other things that have ended up in `app/utils.py`. --- app/__init__.py | 353 ++------------- app/formatters.py | 417 ++++++++++++++++++ app/main/forms.py | 4 +- app/main/views/add_service.py | 3 +- app/main/views/api_keys.py | 3 +- app/main/views/dashboard.py | 3 +- app/main/views/jobs.py | 2 +- app/main/views/manage_users.py | 3 +- app/main/views/service_settings.py | 2 +- app/models/broadcast_message.py | 2 +- app/models/event.py | 2 +- app/utils.py | 110 +---- tests/app/main/views/test_service_settings.py | 2 +- .../views/uploads/test_upload_contact_list.py | 2 +- .../app/main/views/uploads/test_upload_hub.py | 2 +- .../main/views/uploads/test_upload_letter.py | 2 +- tests/app/test_utils.py | 3 +- 17 files changed, 473 insertions(+), 442 deletions(-) create mode 100644 app/formatters.py diff --git a/app/__init__.py b/app/__init__.py index e51356eaa..d0bb4a387 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -1,15 +1,10 @@ import os import pathlib -import re -import urllib -from datetime import datetime, timedelta, timezone from functools import partial from time import monotonic -import humanize import jinja2 from flask import ( - Markup, current_app, flash, g, @@ -20,7 +15,6 @@ from flask import ( session, url_for, ) -from flask._compat import string_types from flask.globals import _lookup_req_object, _request_ctx_stack from flask_login import LoginManager, current_user from flask_wtf import CSRFProtect @@ -29,16 +23,10 @@ from gds_metrics import GDSMetrics from govuk_frontend_jinja.flask_ext import init_govuk_frontend from itsdangerous import BadSignature from notifications_python_client.errors import HTTPError -from notifications_utils import formatters, logging, request_helper -from notifications_utils.field import Field -from notifications_utils.recipients import ( - InvalidPhoneError, - format_phone_number_human_readable, - validate_phone_number, -) +from notifications_utils import logging, request_helper +from notifications_utils.formatters import formatted_list, normalise_lines +from notifications_utils.recipients import format_phone_number_human_readable from notifications_utils.sanitise_text import SanitiseASCII -from notifications_utils.take import Take -from notifications_utils.timezones import utc_string_to_aware_gmt_datetime from werkzeug.exceptions import HTTPException as WerkzeugHTTPException from werkzeug.exceptions import abort from werkzeug.local import LocalProxy @@ -48,6 +36,36 @@ from app.asset_fingerprinter import asset_fingerprinter from app.commands import setup_commands from app.config import configs from app.extensions import antivirus_client, redis_client, zendesk_client +from app.formatters import ( + convert_to_boolean, + format_date, + format_date_human, + format_date_normal, + format_date_numeric, + format_date_short, + format_datetime, + format_datetime_24h, + format_datetime_human, + format_datetime_normal, + format_datetime_relative, + format_datetime_short, + format_day_of_week, + format_delta, + format_delta_days, + format_list_items, + format_notification_status, + format_notification_status_as_field_status, + format_notification_status_as_time, + format_notification_status_as_url, + format_notification_type, + format_number_in_pounds_as_currency, + format_thousands, + format_time, + id_safe, + linkable_name, + nl2br, + valid_phone_number, +) from app.models.organisation import Organisation from app.models.service import Service from app.models.user import AnonymousUser, User @@ -95,7 +113,7 @@ from app.url_converters import ( TemplateTypeConverter, TicketTypeConverter, ) -from app.utils import format_thousands, get_logo_cdn_domain, id_safe +from app.utils import get_logo_cdn_domain login_manager = LoginManager() csrf = CSRFProtect() @@ -245,305 +263,6 @@ def init_app(application): application.url_map.converters['simple_date'] = SimpleDateTypeConverter -def convert_to_boolean(value): - if isinstance(value, string_types): - if value.lower() in ['t', 'true', 'on', 'yes', '1']: - return True - elif value.lower() in ['f', 'false', 'off', 'no', '0']: - return False - - return value - - -def linkable_name(value): - return urllib.parse.quote_plus(value) - - -def format_datetime(date): - return '{} at {}'.format( - format_date(date), - format_time(date) - ) - - -def format_datetime_24h(date): - return '{} at {}'.format( - format_date(date), - format_time_24h(date), - ) - - -def format_datetime_normal(date): - return '{} at {}'.format( - format_date_normal(date), - format_time(date) - ) - - -def format_datetime_short(date): - return '{} at {}'.format( - format_date_short(date), - format_time(date) - ) - - -def format_datetime_relative(date): - return '{} at {}'.format( - get_human_day(date), - format_time(date) - ) - - -def format_datetime_numeric(date): - return '{} {}'.format( - format_date_numeric(date), - format_time_24h(date), - ) - - -def format_date_numeric(date): - return utc_string_to_aware_gmt_datetime(date).strftime('%Y-%m-%d') - - -def format_time_24h(date): - return utc_string_to_aware_gmt_datetime(date).strftime('%H:%M') - - -def get_human_day(time, date_prefix=''): - - # Add 1 minute to transform 00:00 into ‘midnight today’ instead of ‘midnight tomorrow’ - date = (utc_string_to_aware_gmt_datetime(time) - timedelta(minutes=1)).date() - now = datetime.utcnow() - - if date == (now + timedelta(days=1)).date(): - return 'tomorrow' - if date == now.date(): - return 'today' - if date == (now - timedelta(days=1)).date(): - return 'yesterday' - if date.strftime('%Y') != now.strftime('%Y'): - return '{} {} {}'.format( - date_prefix, - _format_datetime_short(date), - date.strftime('%Y'), - ).strip() - return '{} {}'.format( - date_prefix, - _format_datetime_short(date), - ).strip() - - -def format_time(date): - return { - '12:00AM': 'Midnight', - '12:00PM': 'Midday' - }.get( - utc_string_to_aware_gmt_datetime(date).strftime('%-I:%M%p'), - utc_string_to_aware_gmt_datetime(date).strftime('%-I:%M%p') - ).lower() - - -def format_date(date): - return utc_string_to_aware_gmt_datetime(date).strftime('%A %d %B %Y') - - -def format_date_normal(date): - return utc_string_to_aware_gmt_datetime(date).strftime('%d %B %Y').lstrip('0') - - -def format_date_short(date): - return _format_datetime_short(utc_string_to_aware_gmt_datetime(date)) - - -def format_date_human(date): - return get_human_day(date) - - -def format_datetime_human(date, date_prefix=''): - return '{} at {}'.format( - get_human_day(date, date_prefix='on'), - format_time(date), - ) - - -def format_day_of_week(date): - return utc_string_to_aware_gmt_datetime(date).strftime('%A') - - -def _format_datetime_short(datetime): - return datetime.strftime('%d %B').lstrip('0') - - -def naturaltime_without_indefinite_article(date): - return re.sub( - 'an? (.*) ago', - lambda match: '1 {} ago'.format(match.group(1)), - humanize.naturaltime(date), - ) - - -def format_delta(date): - delta = ( - datetime.now(timezone.utc) - ) - ( - utc_string_to_aware_gmt_datetime(date) - ) - if delta < timedelta(seconds=30): - return "just now" - if delta < timedelta(seconds=60): - return "in the last minute" - return naturaltime_without_indefinite_article(delta) - - -def format_delta_days(date): - now = datetime.now(timezone.utc) - date = utc_string_to_aware_gmt_datetime(date) - if date.strftime('%Y-%m-%d') == now.strftime('%Y-%m-%d'): - return "today" - if date.strftime('%Y-%m-%d') == (now - timedelta(days=1)).strftime('%Y-%m-%d'): - return "yesterday" - return naturaltime_without_indefinite_article(now - date) - - -def valid_phone_number(phone_number): - try: - validate_phone_number(phone_number) - return True - except InvalidPhoneError: - return False - - -def format_notification_type(notification_type): - return { - 'email': 'Email', - 'sms': 'Text message', - 'letter': 'Letter' - }[notification_type] - - -def format_notification_status(status, template_type): - return { - 'email': { - 'failed': 'Failed', - 'technical-failure': 'Technical failure', - 'temporary-failure': 'Inbox not accepting messages right now', - 'permanent-failure': 'Email address does not exist', - 'delivered': 'Delivered', - 'sending': 'Sending', - 'created': 'Sending', - 'sent': 'Delivered' - }, - 'sms': { - 'failed': 'Failed', - 'technical-failure': 'Technical failure', - 'temporary-failure': 'Phone not accepting messages right now', - 'permanent-failure': 'Not delivered', - 'delivered': 'Delivered', - 'sending': 'Sending', - 'created': 'Sending', - 'pending': 'Sending', - 'sent': 'Sent to an international number' - }, - 'letter': { - 'failed': '', - 'technical-failure': 'Technical failure', - 'temporary-failure': '', - 'permanent-failure': '', - 'delivered': '', - 'received': '', - 'accepted': '', - 'sending': '', - 'created': '', - 'sent': '', - 'pending-virus-check': '', - 'virus-scan-failed': 'Virus detected', - 'returned-letter': '', - 'cancelled': '', - 'validation-failed': 'Validation failed', - } - }[template_type].get(status, status) - - -def format_notification_status_as_time(status, created, updated): - return dict.fromkeys( - {'created', 'pending', 'sending'}, ' since {}'.format(created) - ).get(status, updated) - - -def format_notification_status_as_field_status(status, notification_type): - return { - 'letter': { - 'failed': 'error', - 'technical-failure': 'error', - 'temporary-failure': 'error', - 'permanent-failure': 'error', - 'delivered': None, - 'sent': None, - 'sending': None, - 'created': None, - 'accepted': None, - 'pending-virus-check': None, - 'virus-scan-failed': 'error', - 'returned-letter': None, - 'cancelled': 'error', - }, - }.get( - notification_type, - { - 'failed': 'error', - 'technical-failure': 'error', - 'temporary-failure': 'error', - 'permanent-failure': 'error', - 'delivered': None, - 'sent': 'sent-international' if notification_type == 'sms' else None, - 'sending': 'default', - 'created': 'default', - 'pending': 'default', - } - ).get(status, 'error') - - -def format_notification_status_as_url(status, notification_type): - url = partial(url_for, "main.message_status") - - if status not in { - 'technical-failure', 'temporary-failure', 'permanent-failure', - }: - return None - - return { - 'email': url(_anchor='email-statuses'), - 'sms': url(_anchor='sms-statuses') - }.get(notification_type) - - -def nl2br(value): - if value: - return Markup(Take(Field( - value, - html='escape', - )).then( - formatters.nl2br - )) - return '' - - -def format_number_in_pounds_as_currency(number): - if number >= 1: - return f"£{number:,.2f}" - - return f"{number * 100:.0f}p" - - -def format_list_items(items, format_string, *args, **kwargs): - """ - Apply formatting to each item in an iterable. Returns a list. - Each item is made available in the format_string as the 'item' keyword argument. - example usage: ['png','svg','pdf']|format_list_items('{0}. {item}', [1,2,3]) -> ['1. png', '2. svg', '3. pdf'] - """ - return [format_string.format(*args, item=item, **kwargs) for item in items] - - @login_manager.user_loader def load_user(user_id): return User.from_id(user_id) @@ -821,8 +540,8 @@ def add_template_filters(application): format_notification_status_as_field_status, format_notification_status_as_url, format_number_in_pounds_as_currency, - formatters.formatted_list, - formatters.normalise_lines, + formatted_list, + normalise_lines, nl2br, format_phone_number_human_readable, format_thousands, diff --git a/app/formatters.py b/app/formatters.py new file mode 100644 index 000000000..bf36351c8 --- /dev/null +++ b/app/formatters.py @@ -0,0 +1,417 @@ +import re +import unicodedata +import urllib +from datetime import datetime, timedelta, timezone +from functools import partial +from math import floor, log10 +from numbers import Number + +import ago +import dateutil +import humanize +from flask import Markup, url_for +from flask._compat import string_types +from notifications_utils.field import Field +from notifications_utils.formatters import make_quotes_smart +from notifications_utils.formatters import nl2br as utils_nl2br +from notifications_utils.recipients import ( + InvalidPhoneError, + validate_phone_number, +) +from notifications_utils.take import Take +from notifications_utils.timezones import utc_string_to_aware_gmt_datetime + + +def convert_to_boolean(value): + if isinstance(value, string_types): + if value.lower() in ['t', 'true', 'on', 'yes', '1']: + return True + elif value.lower() in ['f', 'false', 'off', 'no', '0']: + return False + + return value + + +def format_datetime(date): + return '{} at {}'.format( + format_date(date), + format_time(date) + ) + + +def format_datetime_24h(date): + return '{} at {}'.format( + format_date(date), + format_time_24h(date), + ) + + +def format_datetime_normal(date): + return '{} at {}'.format( + format_date_normal(date), + format_time(date) + ) + + +def format_datetime_short(date): + return '{} at {}'.format( + format_date_short(date), + format_time(date) + ) + + +def format_datetime_relative(date): + return '{} at {}'.format( + get_human_day(date), + format_time(date) + ) + + +def format_datetime_numeric(date): + return '{} {}'.format( + format_date_numeric(date), + format_time_24h(date), + ) + + +def format_date_numeric(date): + return utc_string_to_aware_gmt_datetime(date).strftime('%Y-%m-%d') + + +def format_time_24h(date): + return utc_string_to_aware_gmt_datetime(date).strftime('%H:%M') + + +def get_human_day(time, date_prefix=''): + + # Add 1 minute to transform 00:00 into ‘midnight today’ instead of ‘midnight tomorrow’ + date = (utc_string_to_aware_gmt_datetime(time) - timedelta(minutes=1)).date() + now = datetime.utcnow() + + if date == (now + timedelta(days=1)).date(): + return 'tomorrow' + if date == now.date(): + return 'today' + if date == (now - timedelta(days=1)).date(): + return 'yesterday' + if date.strftime('%Y') != now.strftime('%Y'): + return '{} {} {}'.format( + date_prefix, + _format_datetime_short(date), + date.strftime('%Y'), + ).strip() + return '{} {}'.format( + date_prefix, + _format_datetime_short(date), + ).strip() + + +def format_time(date): + return { + '12:00AM': 'Midnight', + '12:00PM': 'Midday' + }.get( + utc_string_to_aware_gmt_datetime(date).strftime('%-I:%M%p'), + utc_string_to_aware_gmt_datetime(date).strftime('%-I:%M%p') + ).lower() + + +def format_date(date): + return utc_string_to_aware_gmt_datetime(date).strftime('%A %d %B %Y') + + +def format_date_normal(date): + return utc_string_to_aware_gmt_datetime(date).strftime('%d %B %Y').lstrip('0') + + +def format_date_short(date): + return _format_datetime_short(utc_string_to_aware_gmt_datetime(date)) + + +def format_date_human(date): + return get_human_day(date) + + +def format_datetime_human(date, date_prefix=''): + return '{} at {}'.format( + get_human_day(date, date_prefix='on'), + format_time(date), + ) + + +def format_day_of_week(date): + return utc_string_to_aware_gmt_datetime(date).strftime('%A') + + +def _format_datetime_short(datetime): + return datetime.strftime('%d %B').lstrip('0') + + +def naturaltime_without_indefinite_article(date): + return re.sub( + 'an? (.*) ago', + lambda match: '1 {} ago'.format(match.group(1)), + humanize.naturaltime(date), + ) + + +def format_delta(date): + delta = ( + datetime.now(timezone.utc) + ) - ( + utc_string_to_aware_gmt_datetime(date) + ) + if delta < timedelta(seconds=30): + return "just now" + if delta < timedelta(seconds=60): + return "in the last minute" + return naturaltime_without_indefinite_article(delta) + + +def format_delta_days(date): + now = datetime.now(timezone.utc) + date = utc_string_to_aware_gmt_datetime(date) + if date.strftime('%Y-%m-%d') == now.strftime('%Y-%m-%d'): + return "today" + if date.strftime('%Y-%m-%d') == (now - timedelta(days=1)).strftime('%Y-%m-%d'): + return "yesterday" + return naturaltime_without_indefinite_article(now - date) + + +def valid_phone_number(phone_number): + try: + validate_phone_number(phone_number) + return True + except InvalidPhoneError: + return False + + +def format_notification_type(notification_type): + return { + 'email': 'Email', + 'sms': 'Text message', + 'letter': 'Letter' + }[notification_type] + + +def format_notification_status(status, template_type): + return { + 'email': { + 'failed': 'Failed', + 'technical-failure': 'Technical failure', + 'temporary-failure': 'Inbox not accepting messages right now', + 'permanent-failure': 'Email address does not exist', + 'delivered': 'Delivered', + 'sending': 'Sending', + 'created': 'Sending', + 'sent': 'Delivered' + }, + 'sms': { + 'failed': 'Failed', + 'technical-failure': 'Technical failure', + 'temporary-failure': 'Phone not accepting messages right now', + 'permanent-failure': 'Not delivered', + 'delivered': 'Delivered', + 'sending': 'Sending', + 'created': 'Sending', + 'pending': 'Sending', + 'sent': 'Sent to an international number' + }, + 'letter': { + 'failed': '', + 'technical-failure': 'Technical failure', + 'temporary-failure': '', + 'permanent-failure': '', + 'delivered': '', + 'received': '', + 'accepted': '', + 'sending': '', + 'created': '', + 'sent': '', + 'pending-virus-check': '', + 'virus-scan-failed': 'Virus detected', + 'returned-letter': '', + 'cancelled': '', + 'validation-failed': 'Validation failed', + } + }[template_type].get(status, status) + + +def format_notification_status_as_time(status, created, updated): + return dict.fromkeys( + {'created', 'pending', 'sending'}, ' since {}'.format(created) + ).get(status, updated) + + +def format_notification_status_as_field_status(status, notification_type): + return { + 'letter': { + 'failed': 'error', + 'technical-failure': 'error', + 'temporary-failure': 'error', + 'permanent-failure': 'error', + 'delivered': None, + 'sent': None, + 'sending': None, + 'created': None, + 'accepted': None, + 'pending-virus-check': None, + 'virus-scan-failed': 'error', + 'returned-letter': None, + 'cancelled': 'error', + }, + }.get( + notification_type, + { + 'failed': 'error', + 'technical-failure': 'error', + 'temporary-failure': 'error', + 'permanent-failure': 'error', + 'delivered': None, + 'sent': 'sent-international' if notification_type == 'sms' else None, + 'sending': 'default', + 'created': 'default', + 'pending': 'default', + } + ).get(status, 'error') + + +def format_notification_status_as_url(status, notification_type): + url = partial(url_for, "main.message_status") + + if status not in { + 'technical-failure', 'temporary-failure', 'permanent-failure', + }: + return None + + return { + 'email': url(_anchor='email-statuses'), + 'sms': url(_anchor='sms-statuses') + }.get(notification_type) + + +def nl2br(value): + if value: + return Markup(Take(Field( + value, + html='escape', + )).then( + utils_nl2br + )) + return '' + + +def format_number_in_pounds_as_currency(number): + if number >= 1: + return f"£{number:,.2f}" + + return f"{number * 100:.0f}p" + + +def format_list_items(items, format_string, *args, **kwargs): + """ + Apply formatting to each item in an iterable. Returns a list. + Each item is made available in the format_string as the 'item' keyword argument. + example usage: ['png','svg','pdf']|format_list_items('{0}. {item}', [1,2,3]) -> ['1. png', '2. svg', '3. pdf'] + """ + return [format_string.format(*args, item=item, **kwargs) for item in items] + + +def linkable_name(value): + return urllib.parse.quote_plus(value) + + +def format_thousands(value): + if isinstance(value, Number): + return '{:,.0f}'.format(value) + if value is None: + return '' + return value + + +def email_safe(string, whitespace='.'): + # strips accents, diacritics etc + string = ''.join(c for c in unicodedata.normalize('NFD', string) if unicodedata.category(c) != 'Mn') + string = ''.join( + word.lower() if word.isalnum() or word == whitespace else '' + for word in re.sub(r'\s+', whitespace, string.strip()) + ) + string = re.sub(r'\.{2,}', '.', string) + return string.strip('.') + + +def id_safe(string): + return email_safe(string, whitespace='-') + + +def round_to_significant_figures(value, number_of_significant_figures): + if value == 0: + return value + return int(round( + value, + number_of_significant_figures - int(floor(log10(abs(value)))) - 1 + )) + + +def redact_mobile_number(mobile_number, spacing=""): + indices = [-4, -5, -6, -7] + redact_character = spacing + "•" + spacing + mobile_number_list = list(mobile_number.replace(" ", "")) + for i in indices: + mobile_number_list[i] = redact_character + return "".join(mobile_number_list) + + +def get_time_left(created_at, service_data_retention_days=7): + return ago.human( + ( + datetime.now(timezone.utc) + ) - ( + dateutil.parser.parse(created_at).replace(hour=0, minute=0, second=0) + timedelta( + days=service_data_retention_days + 1 + ) + ), + future_tense='Data available for {}', + past_tense='Data no longer available', # No-one should ever see this + precision=1 + ) + + +def starts_with_initial(name): + return bool(re.match(r'^.\.', name)) + + +def remove_middle_initial(name): + return re.sub(r'\s+.\s+', ' ', name) + + +def remove_digits(name): + return ''.join(c for c in name if not c.isdigit()) + + +def normalize_spaces(name): + return ' '.join(name.split()) + + +def guess_name_from_email_address(email_address): + + possible_name = re.split(r'[\@\+]', email_address)[0] + + if '.' not in possible_name or starts_with_initial(possible_name): + return '' + + return Take( + possible_name + ).then( + str.replace, '.', ' ' + ).then( + remove_digits + ).then( + remove_middle_initial + ).then( + str.title + ).then( + make_quotes_smart + ).then( + normalize_spaces + ) diff --git a/app/main/forms.py b/app/main/forms.py index 823137497..8d0b14427 100644 --- a/app/main/forms.py +++ b/app/main/forms.py @@ -38,7 +38,7 @@ from wtforms import ( from wtforms.fields.html5 import EmailField, SearchField, TelField from wtforms.validators import URL, DataRequired, Length, Optional, Regexp -from app import format_thousands +from app.formatters import format_thousands, guess_name_from_email_address from app.main.validators import ( BroadcastLength, CommonlyUsedPassword, @@ -60,7 +60,7 @@ from app.models.roles_and_permissions import ( permissions, roles, ) -from app.utils import guess_name_from_email_address, merge_jsonlike +from app.utils import merge_jsonlike def get_time_value_and_label(future_time): diff --git a/app/main/views/add_service.py b/app/main/views/add_service.py index 0db2ffcc1..62337f2a1 100644 --- a/app/main/views/add_service.py +++ b/app/main/views/add_service.py @@ -3,9 +3,10 @@ from flask_login import current_user from notifications_python_client.errors import HTTPError from app import billing_api_client, service_api_client +from app.formatters import email_safe from app.main import main from app.main.forms import CreateNhsServiceForm, CreateServiceForm -from app.utils import email_safe, user_is_gov_user, user_is_logged_in +from app.utils import user_is_gov_user, user_is_logged_in def _create_service(service_name, organisation_type, email_from, form): diff --git a/app/main/views/api_keys.py b/app/main/views/api_keys.py index 5c599d84f..dac37183c 100644 --- a/app/main/views/api_keys.py +++ b/app/main/views/api_keys.py @@ -15,6 +15,7 @@ from app import ( notification_api_client, service_api_client, ) +from app.formatters import email_safe from app.main import main from app.main.forms import CallbackForm, CreateKeyForm, GuestList from app.notify_client.api_key_api_client import ( @@ -22,7 +23,7 @@ from app.notify_client.api_key_api_client import ( KEY_TYPE_TEAM, KEY_TYPE_TEST, ) -from app.utils import email_safe, user_has_permissions +from app.utils import user_has_permissions dummy_bearer_token = 'bearer_token_set' diff --git a/app/main/views/dashboard.py b/app/main/views/dashboard.py index a634f1a0e..b57d79a2e 100644 --- a/app/main/views/dashboard.py +++ b/app/main/views/dashboard.py @@ -20,11 +20,10 @@ from werkzeug.utils import redirect from app import ( billing_api_client, current_service, - format_date_numeric, - format_datetime_numeric, service_api_client, template_statistics_client, ) +from app.formatters import format_date_numeric, format_datetime_numeric from app.main import main from app.statistics_utils import get_formatted_percentage from app.utils import ( diff --git a/app/main/views/jobs.py b/app/main/views/jobs.py index 36b83bf4d..393ab813e 100644 --- a/app/main/views/jobs.py +++ b/app/main/views/jobs.py @@ -29,6 +29,7 @@ from app import ( notification_api_client, service_api_client, ) +from app.formatters import get_time_left from app.main import main from app.main.forms import SearchNotificationsForm from app.models.job import Job @@ -38,7 +39,6 @@ from app.utils import ( generate_previous_dict, get_letter_printing_statement, get_page_from_request, - get_time_left, parse_filter_args, printing_today_or_tomorrow, set_status_filters, diff --git a/app/main/views/manage_users.py b/app/main/views/manage_users.py index 399bc1ac2..ef380d4f6 100644 --- a/app/main/views/manage_users.py +++ b/app/main/views/manage_users.py @@ -16,6 +16,7 @@ from app.event_handlers import ( create_mobile_number_change_event, create_remove_user_from_service_event, ) +from app.formatters import redact_mobile_number from app.main import main from app.main.forms import ( BroadcastInviteUserForm, @@ -29,7 +30,7 @@ from app.main.forms import ( ) from app.models.roles_and_permissions import broadcast_permissions, permissions from app.models.user import InvitedUser, User -from app.utils import is_gov_user, redact_mobile_number, user_has_permissions +from app.utils import is_gov_user, user_has_permissions @main.route("/services//users") diff --git a/app/main/views/service_settings.py b/app/main/views/service_settings.py index 2dd7a5436..b11345304 100644 --- a/app/main/views/service_settings.py +++ b/app/main/views/service_settings.py @@ -29,6 +29,7 @@ from app import ( user_api_client, ) from app.extensions import zendesk_client +from app.formatters import email_safe from app.main import main from app.main.forms import ( BrandingOptions, @@ -59,7 +60,6 @@ from app.utils import ( DELIVERED_STATUSES, FAILURE_STATUSES, SENDING_STATUSES, - email_safe, user_has_permissions, user_is_gov_user, user_is_platform_admin, diff --git a/app/models/broadcast_message.py b/app/models/broadcast_message.py index bffa1dcb4..337aa7683 100644 --- a/app/models/broadcast_message.py +++ b/app/models/broadcast_message.py @@ -7,13 +7,13 @@ from werkzeug.utils import cached_property from app.broadcast_areas import broadcast_area_libraries from app.broadcast_areas.polygons import Polygons +from app.formatters import round_to_significant_figures from app.models import JSONModel, ModelList from app.models.user import User from app.notify_client.broadcast_message_api_client import ( broadcast_message_api_client, ) from app.notify_client.service_api_client import service_api_client -from app.utils import round_to_significant_figures class BroadcastMessage(JSONModel): diff --git a/app/models/event.py b/app/models/event.py index de3d635cd..0a778fe24 100644 --- a/app/models/event.py +++ b/app/models/event.py @@ -2,9 +2,9 @@ from abc import ABC, abstractmethod from notifications_utils.formatters import formatted_list +from app.formatters import format_thousands from app.models import ModelList from app.notify_client.service_api_client import service_api_client -from app.utils import format_thousands class Event(ABC): diff --git a/app/utils.py b/app/utils.py index 92b48fd2a..41cdc29d1 100644 --- a/app/utils.py +++ b/app/utils.py @@ -1,15 +1,9 @@ import os -import re -import unicodedata -from datetime import datetime, timedelta, timezone +from datetime import datetime, timedelta from functools import wraps from itertools import chain -from math import floor, log10 -from numbers import Number from urllib.parse import urlparse -import ago -import dateutil import pytz from dateutil import parser from flask import ( @@ -24,14 +18,10 @@ from flask import ( ) from flask_login import current_user, login_required from notifications_utils.field import Field -from notifications_utils.formatters import ( - make_quotes_smart, - unescaped_formatted_list, -) +from notifications_utils.formatters import unescaped_formatted_list from notifications_utils.letter_timings import letter_can_be_cancelled from notifications_utils.postal_address import PostalAddress from notifications_utils.recipients import RecipientCSV -from notifications_utils.take import Take from notifications_utils.template import ( BroadcastPreviewTemplate, EmailPreviewTemplate, @@ -270,21 +260,6 @@ def generate_previous_next_dict(view, service_id, page, title, url_args): } -def email_safe(string, whitespace='.'): - # strips accents, diacritics etc - string = ''.join(c for c in unicodedata.normalize('NFD', string) if unicodedata.category(c) != 'Mn') - string = ''.join( - word.lower() if word.isalnum() or word == whitespace else '' - for word in re.sub(r'\s+', whitespace, string.strip()) - ) - string = re.sub(r'\.{2,}', '.', string) - return string.strip('.') - - -def id_safe(string): - return email_safe(string, whitespace='-') - - def get_help_argument(): return request.args.get('help') if request.args.get('help') in ('1', '2', '3') else None @@ -365,21 +340,6 @@ def get_current_financial_year(): return current_year if current_month > 3 else current_year - 1 -def get_time_left(created_at, service_data_retention_days=7): - return ago.human( - ( - datetime.now(timezone.utc) - ) - ( - dateutil.parser.parse(created_at).replace(hour=0, minute=0, second=0) + timedelta( - days=service_data_retention_days + 1 - ) - ), - future_tense='Data available for {}', - past_tense='Data no longer available', # No-one should ever see this - precision=1 - ) - - def get_logo_cdn_domain(): parsed_uri = urlparse(current_app.config['ADMIN_BASE_URL']) @@ -421,46 +381,6 @@ def unicode_truncate(s, length): return encoded.decode('utf-8', 'ignore') -def starts_with_initial(name): - return bool(re.match(r'^.\.', name)) - - -def remove_middle_initial(name): - return re.sub(r'\s+.\s+', ' ', name) - - -def remove_digits(name): - return ''.join(c for c in name if not c.isdigit()) - - -def normalize_spaces(name): - return ' '.join(name.split()) - - -def guess_name_from_email_address(email_address): - - possible_name = re.split(r'[\@\+]', email_address)[0] - - if '.' not in possible_name or starts_with_initial(possible_name): - return '' - - return Take( - possible_name - ).then( - str.replace, '.', ' ' - ).then( - remove_digits - ).then( - remove_middle_initial - ).then( - str.title - ).then( - make_quotes_smart - ).then( - normalize_spaces - ) - - def should_skip_template_page(template_type): return ( current_user.has_permissions('send_messages') @@ -488,15 +408,6 @@ def printing_today_or_tomorrow(created_at): return 'tomorrow' -def redact_mobile_number(mobile_number, spacing=""): - indices = [-4, -5, -6, -7] - redact_character = spacing + "•" + spacing - mobile_number_list = list(mobile_number.replace(" ", "")) - for i in indices: - mobile_number_list[i] = redact_character - return "".join(mobile_number_list) - - def get_letter_printing_statement(status, created_at, long_form=True): created_at_dt = parser.parse(created_at).replace(tzinfo=None) if letter_can_be_cancelled(status, created_at_dt): @@ -709,14 +620,6 @@ class PermanentRedirect(RequestRedirect): code = 301 -def format_thousands(value): - if isinstance(value, Number): - return '{:,.0f}'.format(value) - if value is None: - return '' - return value - - def is_less_than_days_ago(date_from_db, number_of_days): return ( datetime.utcnow().astimezone(pytz.utc) - parser.parse(date_from_db) @@ -762,12 +665,3 @@ def merge_jsonlike(source, destination): source[key] = value merge_items(source, destination) - - -def round_to_significant_figures(value, number_of_significant_figures): - if value == 0: - return value - return int(round( - value, - number_of_significant_figures - int(floor(log10(abs(value)))) - 1 - )) diff --git a/tests/app/main/views/test_service_settings.py b/tests/app/main/views/test_service_settings.py index a2d037f10..b392136a0 100644 --- a/tests/app/main/views/test_service_settings.py +++ b/tests/app/main/views/test_service_settings.py @@ -12,7 +12,7 @@ from notifications_python_client.errors import HTTPError from notifications_utils.clients.zendesk.zendesk_client import ZendeskClient import app -from app.utils import email_safe +from app.formatters import email_safe from tests import ( find_element_by_tag_and_partial_text, invite_json, diff --git a/tests/app/main/views/uploads/test_upload_contact_list.py b/tests/app/main/views/uploads/test_upload_contact_list.py index 5896d5da6..9f0b98ce1 100644 --- a/tests/app/main/views/uploads/test_upload_contact_list.py +++ b/tests/app/main/views/uploads/test_upload_contact_list.py @@ -6,7 +6,7 @@ import pytest from flask import url_for from freezegun import freeze_time -from app.utils import normalize_spaces +from app.formatters import normalize_spaces from tests.conftest import SERVICE_ONE_ID diff --git a/tests/app/main/views/uploads/test_upload_hub.py b/tests/app/main/views/uploads/test_upload_hub.py index f9e44fafe..d1ce523c9 100644 --- a/tests/app/main/views/uploads/test_upload_hub.py +++ b/tests/app/main/views/uploads/test_upload_hub.py @@ -4,7 +4,7 @@ import pytest from flask import url_for from freezegun import freeze_time -from app.utils import normalize_spaces +from app.formatters import normalize_spaces from tests.conftest import ( SERVICE_ONE_ID, create_active_caseworking_user, diff --git a/tests/app/main/views/uploads/test_upload_letter.py b/tests/app/main/views/uploads/test_upload_letter.py index cfc8f64fd..ebd206d9e 100644 --- a/tests/app/main/views/uploads/test_upload_letter.py +++ b/tests/app/main/views/uploads/test_upload_letter.py @@ -5,8 +5,8 @@ from botocore.exceptions import ClientError from flask import make_response, url_for from requests import RequestException +from app.formatters import normalize_spaces from app.s3_client.s3_letter_upload_client import LetterMetadata -from app.utils import normalize_spaces from tests.conftest import SERVICE_ONE_ID diff --git a/tests/app/test_utils.py b/tests/app/test_utils.py index abf1591c3..4246b5037 100644 --- a/tests/app/test_utils.py +++ b/tests/app/test_utils.py @@ -10,9 +10,9 @@ from freezegun import freeze_time from notifications_utils.template import Template from app import format_datetime_relative +from app.formatters import email_safe, round_to_significant_figures from app.utils import ( Spreadsheet, - email_safe, generate_next_dict, generate_notifications_csv, generate_previous_dict, @@ -23,7 +23,6 @@ from app.utils import ( is_less_than_days_ago, merge_jsonlike, printing_today_or_tomorrow, - round_to_significant_figures, ) from tests.conftest import fake_uuid