From 64495e8f5a4cda739ab5f010c33e0d823422febe Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Mon, 31 Mar 2025 09:27:52 -0700 Subject: [PATCH] upgrade mistune --- notifications_utils/markdown.py | 534 +++++++++++++++----------------- 1 file changed, 255 insertions(+), 279 deletions(-) diff --git a/notifications_utils/markdown.py b/notifications_utils/markdown.py index 7d2c719e1..57d35ba43 100644 --- a/notifications_utils/markdown.py +++ b/notifications_utils/markdown.py @@ -1,308 +1,284 @@ +import html import re -from itertools import count import mistune -from ordered_set import OrderedSet +from flask import current_app -from notifications_utils import MAGIC_SEQUENCE, magic_sequence_regex from notifications_utils.formatters import create_sanitised_html_for_url LINK_STYLE = "word-wrap: break-word; color: #1D70B8;" -mistune._block_quote_leading_pattern = re.compile(r"^ *\^ ?", flags=re.M) -mistune.BlockGrammar.block_quote = re.compile(r"^( *\^[^\n]+(\n[^\n]+)*\n*)+") -mistune.BlockGrammar.list_block = re.compile( - r"^( *)([•*-]|\d+\.)[\s\S]+?" - r"(?:" - r"\n+(?=\1?(?:[-*_] *){3,}(?:\n+|$))" # hrule - r"|\n+(?=%s)" # def links - r"|\n+(?=%s)" # def footnotes - r"|\n{2,}" - r"(?! )" - r"(?!\1(?:[•*-]|\d+\.) )\n*" - r"|" - r"\s*$)" - % ( - mistune._pure_pattern(mistune.BlockGrammar.def_links), - mistune._pure_pattern(mistune.BlockGrammar.def_footnotes), + +def escape_plus_lists(markdown_text): + return re.sub(r"(?m)^(\+)(?=\s)", r"\\\1", markdown_text) + + +def autolinkify(text): + # url_pattern = re.compile(r"""(?()]+)""") + url_pattern = re.compile( + r"""(?"')\]]+)""", + re.VERBOSE, ) -) -mistune.BlockGrammar.list_item = re.compile( - r"^(( *)(?:[•*-]|\d+\.)[^\n]*" r"(?:\n(?!\2(?:[•*-]|\d+\.))[^\n]*)*)", flags=re.M -) -mistune.BlockGrammar.list_bullet = re.compile(r"^ *(?:[•*-]|\d+\.)") -mistune.InlineGrammar.url = re.compile(r"""^(https?:\/\/[^\s<]+[^<.,:"')\]\s])""") -mistune.InlineLexer.default_rules = list( - OrderedSet(mistune.InlineLexer.default_rules) - - set( - ( - "emphasis", - "double_emphasis", - "strikethrough", - "code", - ) - ) -) -mistune.InlineLexer.inline_html_rules = list( - set(mistune.InlineLexer.inline_html_rules) - - set( - ( - "emphasis", - "double_emphasis", - "strikethrough", - "code", - ) - ) -) + def replacer(match): + url = match.group(0) + return f"[{url}]({url})" + + return url_pattern.sub(replacer, text) -class NotifyLetterMarkdownPreviewRenderer(mistune.Renderer): - # TODO if we start removing the dead code detected by - # the vulture tool (such as the parameter 'language' here) - # it will break all the tests. Need to do some massive - # cleanup apparently, although it's not clear why vulture - # only recently started detecting this. - def block_code(self, code, language=None): # noqa - return code - - def block_quote(self, text): - return text - - def header(self, text, level, raw=None): # noqa - if level == 1: - return super().header(text, 2) - return self.paragraph(text) - - def hrule(self): - return '
 
' - - def paragraph(self, text): - if text.strip(): - return "

{}

".format(text) - return "" +class EmailRenderer(mistune.HTMLRenderer): def table(self, header, body): return "" - def autolink(self, link, is_email=False): - return "{}".format( - link.replace("http://", "").replace("https://", "") + def table_row(self, content): + return "" + + def table_cell(self, content, **kwargs): + return "" + + def heading(self, text, level): + if level == 1: + return ( + '

' + f"{text}

" + ) + return self.paragraph(text) + + def paragraph(self, text): + if text.strip(): + text = html.unescape(text) + return ( + '

' + text + "

" + ) + + def emphasis(self, text): + return f"*{text}*" + + def strong(self, text): + return f"**{text}**" + + def block_code(self, code, info=None): + return code.strip() + + def block_quote(self, text): + return ( + '
' + f"{text}
" ) - def image(self, src, title, alt_text): # noqa + def thematic_break(self): + return '
' + + def codespan(self, text): + return f"`{text}`" + + def linebreak(self): + return "
" + + def newline(self): + return self.linebreak() + + def list(self, text, ordered, level=None, **kwargs): + tag = "ol" if ordered else "ul" + style = "list-style-type: decimal;" if ordered else "list-style-type: disc;" + return ( + '' + '
' + f'<{tag} style="Margin: 0 0 0 20px; padding: 0; {style}">{text}' + "
" + ) + + def list_item(self, text, level=None): + return ( + '
  • ' + text.strip() + "
    • " + ) + + def link(self, link=None, text=None, title=None, url=None, **kwargs): + + href = html.escape( + url or (link if link and link.startswith("http://", "https://") else "") + ) + display_text = text or link or href or "" + title_attr = f' title="{title}"' if title else "" + return f'{display_text}' + + def autolink(self, link, is_email=False): # noqa + + return create_sanitised_html_for_url(link, style=LINK_STYLE) + + def image(self, src, alt="", title=None, url=None): # noqa + current_app.logger.debug(f"src={src} alt={alt} title={title} url={url}") return "" + def strikethrough(self, text): + return ( + '

    ' + f"~~{text}~~" + "

    " + ) + + +class PlainTextRenderer(mistune.HTMLRenderer): + COLUMN_WIDTH = 65 + + def heading(self, text, level): + if level == 1: + return f"\n\n\n{text}\n{'-' * self.COLUMN_WIDTH}" + return self.paragraph(text) + + def paragraph(self, text): + if text.strip(): + return f"\n\n{text}" + return "" + + def thematic_break(self): + return f"\n\n{'=' * self.COLUMN_WIDTH}" + + def block_quote(self, text): + return text + + def block_code(self, code, info=None): + return code.strip() + + def linebreak(self): + return "\n" + + def list(self, text, ordered, level=None, **kwargs): + + if ordered is True: + text = text.replace("•", "1.", 1) + text = text.replace("•", "2.", 1) + text = text.replace("•", "3.", 1) + + # print(f"LIST ordered={ordered} text={text}") + return f"\n{text}" + + def list_item(self, text, ordered=None, level=None): + # print(f"LIST ITEM = {text} ordered={ordered} level {level}") + return f"\n• {text.strip()}" + + def link(self, link=None, text=None, title=None, url=None, **kwargs): + display_text = text or link or url or "" + href = url or link or "" + output = display_text + if title: + output += f" ({title})" + if href: + output += f": {href}" + return output + + def autolink(self, link, is_email=False): + return link + + def image(self, src, alt="", title=None, url=None): + return "" + + def emphasis(self, text): + return f"*{text}*" + + def strong(self, text): + return f"**{text}**" + + def codespan(self, text): + return f"`{text}`" + + def strikethrough(self, text): + return f"~~{text}~~" + + +class PreheaderRenderer(PlainTextRenderer): + + def heading(self, text, level): + return html.unescape(self.paragraph(text)) + + def thematic_break(self): + return "" + + def link(self, link, text=None, title=None, url=None): + return text or link + + def image(self, src, alt="", title=None, url=None): + current_app.logger.debug("src={src} alt={alt} title={title} url={url}") + return "" + + +class LetterPreviewRenderer(mistune.HTMLRenderer): + def heading(self, text, level): + if level == 1: + return super().heading(text, 2) + return self.paragraph(text) + + def paragraph(self, text): + if text.strip(): + return f"

    {text}

    " + return "" + + def block_code(self, code, info=None): + return code.strip() + + def link(self, link, text=None, title=None, url=None): + current_app.logger(f"title={title}") + href = url + display_text = text or link + return f"{display_text}: {href.replace('http://', '').replace('https://', '')}" + + def autolink(self, link, is_email=False): + current_app.logger.debug(f"is_email={is_email}") + return f"{link.replace('http://', '')}.replace(https://', '')" + + def thematic_break(self): + return '
     
    ' + + def image(self, src, alt="", title=None, **kwargs): + return "" + + def block_quote(self, text): + return text + + def list_item(self, text, level=None): + return f"
  • {text.strip()}
    • \n" + + def emphasis(self, text): + return f"*{text}*" + + def strong(self, text): + return f"**{text}**" + + def codespan(self, text): + return f"`{text}`" + def linebreak(self): return "
    " def newline(self): - return self.linebreak() - - def list_item(self, text): - return "
  • {}
    • \n".format(text.strip()) - - def link(self, link, title, content): - return "{}: {}".format(content, self.autolink(link)) - - def footnote_ref(self, key, index): - return "" - - def footnote_item(self, key, text): - return text - - def footnotes(self, text): - return text + return "
    " -class NotifyEmailMarkdownRenderer(NotifyLetterMarkdownPreviewRenderer): - def header(self, text, level, raw=None): # noqa - if level == 1: - return ( - '

    ' - "{}" - "

    " - ).format(text) - return self.paragraph(text) - - def hrule(self): - return '
    ' - - def linebreak(self): - return "
    " - - def list(self, body, ordered=True): - return ( - ( - '' - "" - '" - "" - "
    ' - '
      ' - "{}" - "
    " - "
    " - ).format(body) - if ordered - else ( - '' - "" - '" - "" - "
    ' - '
      ' - "{}" - "
    " - "
    " - ).format(body) - ) - - def list_item(self, text): - return ( - '
  • ' - "{}" - "
    • " - ).format(text.strip()) - - def paragraph(self, text): - if text.strip(): - return ( - '

    {}

    ' - ).format(text) - return "" - - def block_quote(self, text): - return ( - "
    " - "{}" - "
    " - ).format(text) - - def link(self, link, title, content): - return ('{}').format( - LINK_STYLE, - ' href="{}"'.format(link), - ' title="{}"'.format(title) if title else "", - content, - ) - - def autolink(self, link, is_email=False): - if is_email: - return link - return create_sanitised_html_for_url(link, style=LINK_STYLE) - - -class NotifyPlainTextEmailMarkdownRenderer(NotifyEmailMarkdownRenderer): - COLUMN_WIDTH = 65 - - def header(self, text, level, raw=None): # noqa - if level == 1: - return "".join( - ( - self.linebreak() * 3, - text, - self.linebreak(), - "-" * self.COLUMN_WIDTH, - ) - ) - return self.paragraph(text) - - def hrule(self): - return self.paragraph("=" * self.COLUMN_WIDTH) - - def linebreak(self): - return "\n" - - def list(self, body, ordered=True): - def _get_list_marker(): - decimal = count(1) - return lambda _: "{}.".format(next(decimal)) if ordered else "•" - - return "".join( - ( - self.linebreak(), - re.sub( - magic_sequence_regex, - _get_list_marker(), - body, - ), - ) - ) - - def list_item(self, text): - return "".join( - ( - self.linebreak(), - MAGIC_SEQUENCE, - " ", - text.strip(), - ) - ) - - def paragraph(self, text): - if text.strip(): - return "".join( - ( - self.linebreak() * 2, - text, - ) - ) - return "" - - def block_quote(self, text): - return text - - def link(self, link, title, content): - return "".join( - ( - content, - " ({})".format(title) if title else "", - ": ", - link, - ) - ) - - def autolink(self, link, is_email=False): # noqa - return link - - -class NotifyEmailPreheaderMarkdownRenderer(NotifyPlainTextEmailMarkdownRenderer): - def header(self, text, level, raw=None): # noqa - return self.paragraph(text) - - def hrule(self): - return "" - - def link(self, link, title, content): - return "".join( - ( - content, - " ({})".format(title) if title else "", - ) - ) - - -notify_email_markdown = mistune.Markdown( - renderer=NotifyEmailMarkdownRenderer(), - hard_wrap=True, - use_xhtml=False, +_notify_email_markdown = mistune.create_markdown( + renderer=EmailRenderer(), hard_wrap=True ) -notify_plain_text_email_markdown = mistune.Markdown( - renderer=NotifyPlainTextEmailMarkdownRenderer(), - hard_wrap=True, +notify_letter_preview_markdown = mistune.create_markdown( + renderer=LetterPreviewRenderer() ) -notify_email_preheader_markdown = mistune.Markdown( - renderer=NotifyEmailPreheaderMarkdownRenderer(), - hard_wrap=True, -) -notify_letter_preview_markdown = mistune.Markdown( - renderer=NotifyLetterMarkdownPreviewRenderer(), - hard_wrap=True, - use_xhtml=False, +notify_email_preheader_markdown = mistune.create_markdown(renderer=PreheaderRenderer()) +_notify_plain_text_email_markdown = mistune.create_markdown( + renderer=PlainTextRenderer() ) + + +def notify_email_markdown(text): + text = escape_plus_lists(text) + return _notify_email_markdown(autolinkify(text)) + + +def notify_plain_text_email_markdown(text): + text = escape_plus_lists(text) + return _notify_plain_text_email_markdown(text)