merge from main

This commit is contained in:
Kenneth Kehl
2025-04-08 10:05:30 -07:00
18 changed files with 797 additions and 2192 deletions

View File

@@ -1,308 +1,229 @@
import html
import re
from itertools import count
import mistune
from ordered_set import OrderedSet
from flask import current_app
from notifications_utils import MAGIC_SEQUENCE, magic_sequence_regex
from notifications_utils.formatters import create_sanitised_html_for_url
LINK_STYLE = "word-wrap: break-word; color: #1D70B8;"
mistune._block_quote_leading_pattern = re.compile(r"^ *\^ ?", flags=re.M)
mistune.BlockGrammar.block_quote = re.compile(r"^( *\^[^\n]+(\n[^\n]+)*\n*)+")
mistune.BlockGrammar.list_block = re.compile(
r"^( *)([•*-]|\d+\.)[\s\S]+?"
r"(?:"
r"\n+(?=\1?(?:[-*_] *){3,}(?:\n+|$))" # hrule
r"|\n+(?=%s)" # def links
r"|\n+(?=%s)" # def footnotes
r"|\n{2,}"
r"(?! )"
r"(?!\1(?:[•*-]|\d+\.) )\n*"
r"|"
r"\s*$)"
% (
mistune._pure_pattern(mistune.BlockGrammar.def_links),
mistune._pure_pattern(mistune.BlockGrammar.def_footnotes),
def escape_plus_lists(markdown_text):
return re.sub(r"(?m)^(\+)(?=\s)", r"\\\1", markdown_text)
def autolinkify(text):
# url_pattern = re.compile(r"""(?<!\]\()(?<!["'])\b(https?://[^\s<>()]+)""")
url_pattern = re.compile(
r"""(?<!\]\()
(?<!href=["'])
\b(https?://[^\s<>"')\]]+)""",
re.VERBOSE,
)
)
mistune.BlockGrammar.list_item = re.compile(
r"^(( *)(?:[•*-]|\d+\.)[^\n]*" r"(?:\n(?!\2(?:[•*-]|\d+\.))[^\n]*)*)", flags=re.M
)
mistune.BlockGrammar.list_bullet = re.compile(r"^ *(?:[•*-]|\d+\.)")
mistune.InlineGrammar.url = re.compile(r"""^(https?:\/\/[^\s<]+[^<.,:"')\]\s])""")
mistune.InlineLexer.default_rules = list(
OrderedSet(mistune.InlineLexer.default_rules)
- set(
(
"emphasis",
"double_emphasis",
"strikethrough",
"code",
)
)
)
mistune.InlineLexer.inline_html_rules = list(
set(mistune.InlineLexer.inline_html_rules)
- set(
(
"emphasis",
"double_emphasis",
"strikethrough",
"code",
)
)
)
def replacer(match):
url = match.group(0)
return f"[{url}]({url})"
return url_pattern.sub(replacer, text)
class NotifyLetterMarkdownPreviewRenderer(mistune.Renderer):
# TODO if we start removing the dead code detected by
# the vulture tool (such as the parameter 'language' here)
# it will break all the tests. Need to do some massive
# cleanup apparently, although it's not clear why vulture
# only recently started detecting this.
def block_code(self, code, language=None): # noqa
return code
def block_quote(self, text):
return text
def header(self, text, level, raw=None): # noqa
if level == 1:
return super().header(text, 2)
return self.paragraph(text)
def hrule(self):
return '<div class="page-break">&nbsp;</div>'
def paragraph(self, text):
if text.strip():
return "<p>{}</p>".format(text)
return ""
class EmailRenderer(mistune.HTMLRenderer):
def table(self, header, body):
return ""
def autolink(self, link, is_email=False):
return "<strong>{}</strong>".format(
link.replace("http://", "").replace("https://", "")
)
def image(self, src, title, alt_text): # noqa
def table_row(self, content):
return ""
def linebreak(self):
return "<br>"
def newline(self):
return self.linebreak()
def list_item(self, text):
return "<li>{}</li>\n".format(text.strip())
def link(self, link, title, content):
return "{}: {}".format(content, self.autolink(link))
def footnote_ref(self, key, index):
def table_cell(self, content, **kwargs):
return ""
def footnote_item(self, key, text):
return text
def footnotes(self, text):
return text
class NotifyEmailMarkdownRenderer(NotifyLetterMarkdownPreviewRenderer):
def header(self, text, level, raw=None): # noqa
def heading(self, text, level):
if level == 1:
return (
'<h2 style="Margin: 0 0 20px 0; padding: 0; '
'font-size: 27px; line-height: 35px; font-weight: bold; color: #0B0C0C;">'
"{}"
"</h2>"
).format(text)
f"{text}</h2>"
)
return self.paragraph(text)
def hrule(self):
def paragraph(self, text):
if text.strip():
text = html.unescape(text)
return (
'<p style="Margin: 0 0 20px 0; font-size: 19px; '
'line-height: 25px; color: #0B0C0C;">' + text + "</p>"
)
def emphasis(self, text):
return f"*{text}*"
def strong(self, text):
return f"**{text}**"
def block_code(self, code, info=None):
return code.strip()
def block_quote(self, text):
return (
'<blockquote style="Margin: 0 0 20px 0; border-left: 10px solid #B1B4B6; '
'padding: 15 px 0 0.1px 15 px; font-size: 19px; line-height: 25px;">'
f"{text}</blockquote>"
)
def thematic_break(self):
return '<hr style="border: 0; height: 1px; background: #B1B4B6; Margin: 30px 0 30px 0;">'
def codespan(self, text):
return f"`{text}`"
def linebreak(self):
return "<br />"
def list(self, body, ordered=True):
def newline(self):
return self.linebreak()
def list(self, text, ordered, level=None, **kwargs):
tag = "ol" if ordered else "ul"
style = "list-style-type: decimal;" if ordered else "list-style-type: disc;"
return (
(
'<table role="presentation" style="padding: 0 0 20px 0;">'
"<tr>"
'<td style="font-family: Helvetica, Arial, sans-serif;">'
'<ol style="Margin: 0 0 0 20px; padding: 0; list-style-type: decimal;">'
"{}"
"</ol>"
"</td>"
"</tr>"
"</table>"
).format(body)
if ordered
else (
'<table role="presentation" style="padding: 0 0 20px 0;">'
"<tr>"
'<td style="font-family: Helvetica, Arial, sans-serif;">'
'<ul style="Margin: 0 0 0 20px; padding: 0; list-style-type: disc;">'
"{}"
"</ul>"
"</td>"
"</tr>"
"</table>"
).format(body)
'<table role="presentation" style="padding 0 0 20px 0;">'
'<tr><td style="font-family: Helvetica, Arial, sans-serif;">'
f'<{tag} style="Margin: 0 0 0 20px; padding: 0; {style}">{text}</{tag}>'
"</td></tr></table>"
)
def list_item(self, text):
def list_item(self, text, level=None):
return (
'<li style="Margin: 5px 0 5px; padding: 0 0 0 5px; font-size: 19px;'
'line-height: 25px; color: #0B0C0C;">'
"{}"
"</li>"
).format(text.strip())
def paragraph(self, text):
if text.strip():
return (
'<p style="Margin: 0 0 20px 0; font-size: 19px; line-height: 25px; color: #0B0C0C;">{}</p>'
).format(text)
return ""
def block_quote(self, text):
return (
"<blockquote "
'style="Margin: 0 0 20px 0; border-left: 10px solid #B1B4B6;'
'padding: 15px 0 0.1px 15px; font-size: 19px; line-height: 25px;"'
">"
"{}"
"</blockquote>"
).format(text)
def link(self, link, title, content):
return ('<a style="{}"{}{}>{}</a>').format(
LINK_STYLE,
' href="{}"'.format(link),
' title="{}"'.format(title) if title else "",
content,
'line-height: 25px; color: #0B0C0C;">' + text.strip() + "</li>"
)
def autolink(self, link, is_email=False):
if is_email:
return link
def link(self, link=None, text=None, title=None, url=None, **kwargs):
href = html.escape(
url or (link if link and link.startswith("http://", "https://") else "")
)
display_text = text or link or href or ""
title_attr = f' title="{title}"' if title else ""
return f'<a style="{LINK_STYLE}" href="{href}"{title_attr}>{display_text}</a>'
def autolink(self, link, is_email=False): # noqa
return create_sanitised_html_for_url(link, style=LINK_STYLE)
def image(self, src, alt="", title=None, url=None): # noqa
current_app.logger.debug(f"src={src} alt={alt} title={title} url={url}")
return ""
class NotifyPlainTextEmailMarkdownRenderer(NotifyEmailMarkdownRenderer):
def strikethrough(self, text):
return (
'<p style="Margin: 0 0 20px 0; font-size: 19px; line-height: 25px; color: #0B0C0C;">'
f"~~{text}~~"
"</p>"
)
class PlainTextRenderer(mistune.HTMLRenderer):
COLUMN_WIDTH = 65
def header(self, text, level, raw=None): # noqa
def heading(self, text, level):
if level == 1:
return "".join(
(
self.linebreak() * 3,
text,
self.linebreak(),
"-" * self.COLUMN_WIDTH,
)
)
return f"\n\n\n{text}\n{'-' * self.COLUMN_WIDTH}"
return self.paragraph(text)
def hrule(self):
return self.paragraph("=" * self.COLUMN_WIDTH)
def linebreak(self):
return "\n"
def list(self, body, ordered=True):
def _get_list_marker():
decimal = count(1)
return lambda _: "{}.".format(next(decimal)) if ordered else ""
return "".join(
(
self.linebreak(),
re.sub(
magic_sequence_regex,
_get_list_marker(),
body,
),
)
)
def list_item(self, text):
return "".join(
(
self.linebreak(),
MAGIC_SEQUENCE,
" ",
text.strip(),
)
)
def paragraph(self, text):
if text.strip():
return "".join(
(
self.linebreak() * 2,
text,
)
)
return f"\n\n{text}"
return ""
def thematic_break(self):
return f"\n\n{'=' * self.COLUMN_WIDTH}"
def block_quote(self, text):
return text
def link(self, link, title, content):
return "".join(
(
content,
" ({})".format(title) if title else "",
": ",
link,
)
)
def block_code(self, code, info=None):
return code.strip()
def autolink(self, link, is_email=False): # noqa
def linebreak(self):
return "\n"
def list(self, text, ordered, level=None, **kwargs):
if ordered is True:
text = text.replace("", "1.", 1)
text = text.replace("", "2.", 1)
text = text.replace("", "3.", 1)
# print(f"LIST ordered={ordered} text={text}")
return f"\n{text}"
def list_item(self, text, ordered=None, level=None):
# print(f"LIST ITEM = {text} ordered={ordered} level {level}")
return f"\n{text.strip()}"
def link(self, link=None, text=None, title=None, url=None, **kwargs):
display_text = text or link or url or ""
href = url or link or ""
output = display_text
if title:
output += f" ({title})"
if href:
output += f": {href}"
return output
def autolink(self, link, is_email=False):
return link
class NotifyEmailPreheaderMarkdownRenderer(NotifyPlainTextEmailMarkdownRenderer):
def header(self, text, level, raw=None): # noqa
return self.paragraph(text)
def hrule(self):
def image(self, src, alt="", title=None, url=None):
return ""
def link(self, link, title, content):
return "".join(
(
content,
" ({})".format(title) if title else "",
)
)
def emphasis(self, text):
return f"*{text}*"
def strong(self, text):
return f"**{text}**"
def codespan(self, text):
return f"`{text}`"
def strikethrough(self, text):
return f"~~{text}~~"
notify_email_markdown = mistune.Markdown(
renderer=NotifyEmailMarkdownRenderer(),
hard_wrap=True,
use_xhtml=False,
class PreheaderRenderer(PlainTextRenderer):
def heading(self, text, level):
return html.unescape(self.paragraph(text))
def thematic_break(self):
return ""
def link(self, link, text=None, title=None, url=None):
return text or link
def image(self, src, alt="", title=None, url=None):
current_app.logger.debug("src={src} alt={alt} title={title} url={url}")
return ""
_notify_email_markdown = mistune.create_markdown(
renderer=EmailRenderer(), hard_wrap=True
)
notify_plain_text_email_markdown = mistune.Markdown(
renderer=NotifyPlainTextEmailMarkdownRenderer(),
hard_wrap=True,
)
notify_email_preheader_markdown = mistune.Markdown(
renderer=NotifyEmailPreheaderMarkdownRenderer(),
hard_wrap=True,
)
notify_letter_preview_markdown = mistune.Markdown(
renderer=NotifyLetterMarkdownPreviewRenderer(),
hard_wrap=True,
use_xhtml=False,
notify_email_preheader_markdown = mistune.create_markdown(renderer=PreheaderRenderer())
_notify_plain_text_email_markdown = mistune.create_markdown(
renderer=PlainTextRenderer()
)
def notify_email_markdown(text):
text = escape_plus_lists(text)
return _notify_email_markdown(autolinkify(text))
def notify_plain_text_email_markdown(text):
text = escape_plus_lists(text)
return _notify_plain_text_email_markdown(text)

View File

@@ -8,20 +8,13 @@ from os import path
from jinja2 import Environment, FileSystemLoader, select_autoescape
from markupsafe import Markup
from notifications_utils import (
LETTER_MAX_PAGE_COUNT,
MAGIC_SEQUENCE,
SMS_CHAR_COUNT_LIMIT,
utc_now,
)
from notifications_utils.countries.data import Postage
from notifications_utils import MAGIC_SEQUENCE, SMS_CHAR_COUNT_LIMIT
from notifications_utils.field import Field, PlainTextField
from notifications_utils.formatters import (
add_prefix,
add_trailing_newline,
autolink_urls,
escape_html,
formatted_list,
make_quotes_smart,
nl2br,
normalise_multiple_newlines,
@@ -30,7 +23,6 @@ from notifications_utils.formatters import (
remove_smart_quotes_from_email_addresses,
remove_whitespace_before_punctuation,
replace_hyphens_with_en_dashes,
replace_hyphens_with_non_breaking_hyphens,
sms_encode,
strip_leading_whitespace,
strip_unsupported_characters,
@@ -40,10 +32,8 @@ from notifications_utils.insensitive_dict import InsensitiveDict
from notifications_utils.markdown import (
notify_email_markdown,
notify_email_preheader_markdown,
notify_letter_preview_markdown,
notify_plain_text_email_markdown,
)
from notifications_utils.postal_address import PostalAddress, address_lines_1_to_7_keys
from notifications_utils.sanitise_text import SanitiseSMS
from notifications_utils.take import Take
from notifications_utils.template_change import TemplateChange
@@ -714,231 +704,6 @@ class EmailPreviewTemplate(BaseEmailTemplate):
)
class BaseLetterTemplate(SubjectMixin, Template):
template_type = "letter"
address_block = "\n".join(
f'(({line.replace("_", " ")}))' for line in address_lines_1_to_7_keys
)
def __init__(
self,
template,
values=None,
contact_block=None,
admin_base_url="http://localhost:6012",
logo_file_name=None,
redact_missing_personalisation=False,
date=None,
):
self.contact_block = (contact_block or "").strip()
super().__init__(
template,
values,
redact_missing_personalisation=redact_missing_personalisation,
)
self.admin_base_url = admin_base_url
self.logo_file_name = logo_file_name
self.date = date or utc_now()
@property
def subject(self):
return (
Take(
Field(
self._subject,
self.values,
redact_missing_personalisation=self.redact_missing_personalisation,
html="escape",
)
)
.then(do_nice_typography)
.then(normalise_whitespace)
)
@property
def placeholders(self):
return get_placeholders(self.contact_block) | super().placeholders
@property
def postal_address(self):
return PostalAddress.from_personalisation(InsensitiveDict(self.values))
@property
def _address_block(self):
if (
self.postal_address.has_enough_lines
and not self.postal_address.has_too_many_lines
):
return self.postal_address.normalised_lines
if "address line 7" not in self.values and "postcode" in self.values:
self.values["address line 7"] = self.values["postcode"]
return Field(
self.address_block,
self.values,
html="escape",
with_brackets=False,
).splitlines()
@property
def _contact_block(self):
return (
Take(
Field(
"\n".join(line.strip() for line in self.contact_block.split("\n")),
self.values,
redact_missing_personalisation=self.redact_missing_personalisation,
html="escape",
)
)
.then(remove_whitespace_before_punctuation)
.then(nl2br)
)
@property
def _date(self):
return self.date.strftime("%-d %B %Y")
@property
def _message(self):
return (
Take(
Field(
self.content,
self.values,
html="escape",
markdown_lists=True,
redact_missing_personalisation=self.redact_missing_personalisation,
)
)
.then(add_trailing_newline)
.then(notify_letter_preview_markdown)
.then(do_nice_typography)
.then(replace_hyphens_with_non_breaking_hyphens)
)
class LetterPreviewTemplate(BaseLetterTemplate):
jinja_template = template_env.get_template("letter_pdf/preview.jinja2")
def __str__(self):
return Markup(
self.jinja_template.render(
{
"admin_base_url": self.admin_base_url,
"logo_file_name": self.logo_file_name,
# logo_class should only ever be None, svg or png
"logo_class": (
self.logo_file_name.lower()[-3:]
if self.logo_file_name
else None
),
"subject": self.subject,
"message": self._message,
"address": self._address_block,
"contact_block": self._contact_block,
"date": self._date,
}
)
)
class LetterPrintTemplate(LetterPreviewTemplate):
jinja_template = template_env.get_template("letter_pdf/print.jinja2")
class LetterImageTemplate(BaseLetterTemplate):
jinja_template = template_env.get_template("letter_image_template.jinja2")
first_page_number = 1
allowed_postage_types = (
Postage.FIRST,
Postage.SECOND,
Postage.EUROPE,
Postage.REST_OF_WORLD,
)
def __init__(
self,
template,
values=None,
image_url=None,
page_count=None,
contact_block=None,
postage=None,
):
super().__init__(template, values, contact_block=contact_block)
if not image_url:
raise TypeError("image_url is required")
if not page_count:
raise TypeError("page_count is required")
if postage not in [None] + list(self.allowed_postage_types):
raise TypeError(
"postage must be None, {}".format(
formatted_list(
self.allowed_postage_types,
conjunction="or",
before_each="'",
after_each="'",
)
)
)
self.image_url = image_url
self.page_count = int(page_count)
self._postage = postage
@property
def postage(self):
if self.postal_address.international:
return self.postal_address.postage
return self._postage
@property
def last_page_number(self):
return min(self.page_count, LETTER_MAX_PAGE_COUNT) + self.first_page_number
@property
def page_numbers(self):
return list(range(self.first_page_number, self.last_page_number))
@property
def postage_description(self):
return {
Postage.FIRST: "first class",
Postage.SECOND: "second class",
Postage.EUROPE: "international",
Postage.REST_OF_WORLD: "international",
}.get(self.postage)
@property
def postage_class_value(self):
return {
Postage.FIRST: "letter-postage-first",
Postage.SECOND: "letter-postage-second",
Postage.EUROPE: "letter-postage-international",
Postage.REST_OF_WORLD: "letter-postage-international",
}.get(self.postage)
def __str__(self):
return Markup(
self.jinja_template.render(
{
"image_url": self.image_url,
"page_numbers": self.page_numbers,
"address": self._address_block,
"contact_block": self._contact_block,
"date": self._date,
"subject": self.subject,
"message": self._message,
"show_postage": bool(self.postage),
"postage_description": self.postage_description,
"postage_class_value": self.postage_class_value,
}
)
)
def get_sms_fragment_count(character_count, non_gsm_characters):
if non_gsm_characters:
return 1 if character_count <= 70 else math.ceil(float(character_count) / 67)