upgrade mistune

notifications_utils/markdown.py

@@ -1,308 +1,284 @@
+import html
 import re
-from itertools import count
 
 import mistune
-from ordered_set import OrderedSet
+from flask import current_app
 
-from notifications_utils import MAGIC_SEQUENCE, magic_sequence_regex
 from notifications_utils.formatters import create_sanitised_html_for_url
 
 LINK_STYLE = "word-wrap: break-word; color: #1D70B8;"
 
-mistune._block_quote_leading_pattern = re.compile(r"^ *\^ ?", flags=re.M)
+
-mistune.BlockGrammar.block_quote = re.compile(r"^( *\^[^\n]+(\n[^\n]+)*\n*)+")
+def escape_plus_lists(markdown_text):
-mistune.BlockGrammar.list_block = re.compile(
+    return re.sub(r"(?m)^(\+)(?=\s)", r"\\\1", markdown_text)
-    r"^( *)([•*-]|\d+\.)[\s\S]+?"
+
-    r"(?:"
+
-    r"\n+(?=\1?(?:[-*_] *){3,}(?:\n+|$))"  # hrule
+def autolinkify(text):
-    r"|\n+(?=%s)"  # def links
+    # url_pattern = re.compile(r"""(?<!\]\()(?<!["'])\b(https?://[^\s<>()]+)""")
-    r"|\n+(?=%s)"  # def footnotes
+    url_pattern = re.compile(
-    r"|\n{2,}"
+        r"""(?<!\]\()
-    r"(?! )"
+        (?<!href=["'])
-    r"(?!\1(?:[•*-]|\d+\.) )\n*"
+        \b(https?://[^\s<>"')\]]+)""",
-    r"|"
+        re.VERBOSE,
-    r"\s*$)"
-    % (
-        mistune._pure_pattern(mistune.BlockGrammar.def_links),
-        mistune._pure_pattern(mistune.BlockGrammar.def_footnotes),
     )
-)
-mistune.BlockGrammar.list_item = re.compile(
-    r"^(( *)(?:[•*-]|\d+\.)[^\n]*" r"(?:\n(?!\2(?:[•*-]|\d+\.))[^\n]*)*)", flags=re.M
-)
-mistune.BlockGrammar.list_bullet = re.compile(r"^ *(?:[•*-]|\d+\.)")
-mistune.InlineGrammar.url = re.compile(r"""^(https?:\/\/[^\s<]+[^<.,:"')\]\s])""")
 
-mistune.InlineLexer.default_rules = list(
+    def replacer(match):
-    OrderedSet(mistune.InlineLexer.default_rules)
+        url = match.group(0)
-    - set(
+        return f"[{url}]({url})"
-        (
+
-            "emphasis",
+    return url_pattern.sub(replacer, text)
-            "double_emphasis",
-            "strikethrough",
-            "code",
-        )
-    )
-)
-mistune.InlineLexer.inline_html_rules = list(
-    set(mistune.InlineLexer.inline_html_rules)
-    - set(
-        (
-            "emphasis",
-            "double_emphasis",
-            "strikethrough",
-            "code",
-        )
-    )
-)
 
 
-class NotifyLetterMarkdownPreviewRenderer(mistune.Renderer):
+class EmailRenderer(mistune.HTMLRenderer):
-    # TODO if we start removing the dead code detected by
-    # the vulture tool (such as the parameter 'language' here)
-    # it will break all the tests.  Need to do some massive
-    # cleanup apparently, although it's not clear why vulture
-    # only recently started detecting this.
-    def block_code(self, code, language=None):  # noqa
-        return code
-
-    def block_quote(self, text):
-        return text
-
-    def header(self, text, level, raw=None):  # noqa
-        if level == 1:
-            return super().header(text, 2)
-        return self.paragraph(text)
-
-    def hrule(self):
-        return '<div class="page-break">&nbsp;</div>'
-
-    def paragraph(self, text):
-        if text.strip():
-            return "<p>{}</p>".format(text)
-        return ""
 
     def table(self, header, body):
         return ""
 
-    def autolink(self, link, is_email=False):
+    def table_row(self, content):
-        return "<strong>{}</strong>".format(
+        return ""
-            link.replace("http://", "").replace("https://", "")
+
+    def table_cell(self, content, **kwargs):
+        return ""
+
+    def heading(self, text, level):
+        if level == 1:
+            return (
+                '<h2 style="Margin: 0 0 20px 0; padding: 0; '
+                'font-size: 27px; line-height: 35px; font-weight: bold; color: #0B0C0C;">'
+                f"{text}</h2>"
+            )
+        return self.paragraph(text)
+
+    def paragraph(self, text):
+        if text.strip():
+            text = html.unescape(text)
+            return (
+                '<p style="Margin: 0 0 20px 0; font-size: 19px; '
+                'line-height: 25px; color: #0B0C0C;">' + text + "</p>"
+            )
+
+    def emphasis(self, text):
+        return f"*{text}*"
+
+    def strong(self, text):
+        return f"**{text}**"
+
+    def block_code(self, code, info=None):
+        return code.strip()
+
+    def block_quote(self, text):
+        return (
+            '<blockquote style="Margin: 0 0 20px 0; border-left: 10px solid #B1B4B6; '
+            'padding: 15 px 0 0.1px 15 px; font-size: 19px; line-height: 25px;">'
+            f"{text}</blockquote>"
         )
 
-    def image(self, src, title, alt_text):  # noqa
+    def thematic_break(self):
+        return '<hr style="border: 0; height: 1px; background: #B1B4B6; Margin: 30px 0 30px 0;">'
+
+    def codespan(self, text):
+        return f"`{text}`"
+
+    def linebreak(self):
+        return "<br />"
+
+    def newline(self):
+        return self.linebreak()
+
+    def list(self, text, ordered, level=None, **kwargs):
+        tag = "ol" if ordered else "ul"
+        style = "list-style-type: decimal;" if ordered else "list-style-type: disc;"
+        return (
+            '<table role="presentation" style="padding 0 0 20px 0;">'
+            '<tr><td style="font-family: Helvetica, Arial, sans-serif;">'
+            f'<{tag} style="Margin: 0 0 0 20px; padding: 0; {style}">{text}</{tag}>'
+            "</td></tr></table>"
+        )
+
+    def list_item(self, text, level=None):
+        return (
+            '<li style="Margin: 5px 0 5px; padding: 0 0 0 5px; font-size: 19px;'
+            'line-height: 25px; color: #0B0C0C;">' + text.strip() + "</li>"
+        )
+
+    def link(self, link=None, text=None, title=None, url=None, **kwargs):
+
+        href = html.escape(
+            url or (link if link and link.startswith("http://", "https://") else "")
+        )
+        display_text = text or link or href or ""
+        title_attr = f' title="{title}"' if title else ""
+        return f'<a style="{LINK_STYLE}" href="{href}"{title_attr}>{display_text}</a>'
+
+    def autolink(self, link, is_email=False):  # noqa
+
+        return create_sanitised_html_for_url(link, style=LINK_STYLE)
+
+    def image(self, src, alt="", title=None, url=None):  # noqa
+        current_app.logger.debug(f"src={src} alt={alt} title={title} url={url}")
         return ""
 
+    def strikethrough(self, text):
+        return (
+            '<p style="Margin: 0 0 20px 0; font-size: 19px; line-height: 25px; color: #0B0C0C;">'
+            f"~~{text}~~"
+            "</p>"
+        )
+
+
+class PlainTextRenderer(mistune.HTMLRenderer):
+    COLUMN_WIDTH = 65
+
+    def heading(self, text, level):
+        if level == 1:
+            return f"\n\n\n{text}\n{'-' * self.COLUMN_WIDTH}"
+        return self.paragraph(text)
+
+    def paragraph(self, text):
+        if text.strip():
+            return f"\n\n{text}"
+        return ""
+
+    def thematic_break(self):
+        return f"\n\n{'=' * self.COLUMN_WIDTH}"
+
+    def block_quote(self, text):
+        return text
+
+    def block_code(self, code, info=None):
+        return code.strip()
+
+    def linebreak(self):
+        return "\n"
+
+    def list(self, text, ordered, level=None, **kwargs):
+
+        if ordered is True:
+            text = text.replace("•", "1.", 1)
+            text = text.replace("•", "2.", 1)
+            text = text.replace("•", "3.", 1)
+
+        # print(f"LIST ordered={ordered} text={text}")
+        return f"\n{text}"
+
+    def list_item(self, text, ordered=None, level=None):
+        # print(f"LIST ITEM = {text} ordered={ordered} level {level}")
+        return f"\n• {text.strip()}"
+
+    def link(self, link=None, text=None, title=None, url=None, **kwargs):
+        display_text = text or link or url or ""
+        href = url or link or ""
+        output = display_text
+        if title:
+            output += f" ({title})"
+        if href:
+            output += f": {href}"
+        return output
+
+    def autolink(self, link, is_email=False):
+        return link
+
+    def image(self, src, alt="", title=None, url=None):
+        return ""
+
+    def emphasis(self, text):
+        return f"*{text}*"
+
+    def strong(self, text):
+        return f"**{text}**"
+
+    def codespan(self, text):
+        return f"`{text}`"
+
+    def strikethrough(self, text):
+        return f"~~{text}~~"
+
+
+class PreheaderRenderer(PlainTextRenderer):
+
+    def heading(self, text, level):
+        return html.unescape(self.paragraph(text))
+
+    def thematic_break(self):
+        return ""
+
+    def link(self, link, text=None, title=None, url=None):
+        return text or link
+
+    def image(self, src, alt="", title=None, url=None):
+        current_app.logger.debug("src={src} alt={alt} title={title} url={url}")
+        return ""
+
+
+class LetterPreviewRenderer(mistune.HTMLRenderer):
+    def heading(self, text, level):
+        if level == 1:
+            return super().heading(text, 2)
+        return self.paragraph(text)
+
+    def paragraph(self, text):
+        if text.strip():
+            return f"<p>{text}</p>"
+        return ""
+
+    def block_code(self, code, info=None):
+        return code.strip()
+
+    def link(self, link, text=None, title=None, url=None):
+        current_app.logger(f"title={title}")
+        href = url
+        display_text = text or link
+        return f"{display_text}: <strong>{href.replace('http://', '').replace('https://', '')}</strong>"
+
+    def autolink(self, link, is_email=False):
+        current_app.logger.debug(f"is_email={is_email}")
+        return f"<strong>{link.replace('http://', '')}.replace(https://', '')</strong>"
+
+    def thematic_break(self):
+        return '<div class="page-break">&nbsp;</div>'
+
+    def image(self, src, alt="", title=None, **kwargs):
+        return ""
+
+    def block_quote(self, text):
+        return text
+
+    def list_item(self, text, level=None):
+        return f"<li>{text.strip()}</li>\n"
+
+    def emphasis(self, text):
+        return f"*{text}*"
+
+    def strong(self, text):
+        return f"**{text}**"
+
+    def codespan(self, text):
+        return f"`{text}`"
+
     def linebreak(self):
         return "<br>"
 
     def newline(self):
-        return self.linebreak()
+        return "<br>"
-
-    def list_item(self, text):
-        return "<li>{}</li>\n".format(text.strip())
-
-    def link(self, link, title, content):
-        return "{}: {}".format(content, self.autolink(link))
-
-    def footnote_ref(self, key, index):
-        return ""
-
-    def footnote_item(self, key, text):
-        return text
-
-    def footnotes(self, text):
-        return text
 
 
-class NotifyEmailMarkdownRenderer(NotifyLetterMarkdownPreviewRenderer):
+_notify_email_markdown = mistune.create_markdown(
-    def header(self, text, level, raw=None):  # noqa
+    renderer=EmailRenderer(), hard_wrap=True
-        if level == 1:
-            return (
-                '<h2 style="Margin: 0 0 20px 0; padding: 0; '
-                'font-size: 27px; line-height: 35px; font-weight: bold; color: #0B0C0C;">'
-                "{}"
-                "</h2>"
-            ).format(text)
-        return self.paragraph(text)
-
-    def hrule(self):
-        return '<hr style="border: 0; height: 1px; background: #B1B4B6; Margin: 30px 0 30px 0;">'
-
-    def linebreak(self):
-        return "<br />"
-
-    def list(self, body, ordered=True):
-        return (
-            (
-                '<table role="presentation" style="padding: 0 0 20px 0;">'
-                "<tr>"
-                '<td style="font-family: Helvetica, Arial, sans-serif;">'
-                '<ol style="Margin: 0 0 0 20px; padding: 0; list-style-type: decimal;">'
-                "{}"
-                "</ol>"
-                "</td>"
-                "</tr>"
-                "</table>"
-            ).format(body)
-            if ordered
-            else (
-                '<table role="presentation" style="padding: 0 0 20px 0;">'
-                "<tr>"
-                '<td style="font-family: Helvetica, Arial, sans-serif;">'
-                '<ul style="Margin: 0 0 0 20px; padding: 0; list-style-type: disc;">'
-                "{}"
-                "</ul>"
-                "</td>"
-                "</tr>"
-                "</table>"
-            ).format(body)
-        )
-
-    def list_item(self, text):
-        return (
-            '<li style="Margin: 5px 0 5px; padding: 0 0 0 5px; font-size: 19px;'
-            'line-height: 25px; color: #0B0C0C;">'
-            "{}"
-            "</li>"
-        ).format(text.strip())
-
-    def paragraph(self, text):
-        if text.strip():
-            return (
-                '<p style="Margin: 0 0 20px 0; font-size: 19px; line-height: 25px; color: #0B0C0C;">{}</p>'
-            ).format(text)
-        return ""
-
-    def block_quote(self, text):
-        return (
-            "<blockquote "
-            'style="Margin: 0 0 20px 0; border-left: 10px solid #B1B4B6;'
-            'padding: 15px 0 0.1px 15px; font-size: 19px; line-height: 25px;"'
-            ">"
-            "{}"
-            "</blockquote>"
-        ).format(text)
-
-    def link(self, link, title, content):
-        return ('<a style="{}"{}{}>{}</a>').format(
-            LINK_STYLE,
-            ' href="{}"'.format(link),
-            ' title="{}"'.format(title) if title else "",
-            content,
-        )
-
-    def autolink(self, link, is_email=False):
-        if is_email:
-            return link
-        return create_sanitised_html_for_url(link, style=LINK_STYLE)
-
-
-class NotifyPlainTextEmailMarkdownRenderer(NotifyEmailMarkdownRenderer):
-    COLUMN_WIDTH = 65
-
-    def header(self, text, level, raw=None):  # noqa
-        if level == 1:
-            return "".join(
-                (
-                    self.linebreak() * 3,
-                    text,
-                    self.linebreak(),
-                    "-" * self.COLUMN_WIDTH,
-                )
-            )
-        return self.paragraph(text)
-
-    def hrule(self):
-        return self.paragraph("=" * self.COLUMN_WIDTH)
-
-    def linebreak(self):
-        return "\n"
-
-    def list(self, body, ordered=True):
-        def _get_list_marker():
-            decimal = count(1)
-            return lambda _: "{}.".format(next(decimal)) if ordered else "•"
-
-        return "".join(
-            (
-                self.linebreak(),
-                re.sub(
-                    magic_sequence_regex,
-                    _get_list_marker(),
-                    body,
-                ),
-            )
-        )
-
-    def list_item(self, text):
-        return "".join(
-            (
-                self.linebreak(),
-                MAGIC_SEQUENCE,
-                " ",
-                text.strip(),
-            )
-        )
-
-    def paragraph(self, text):
-        if text.strip():
-            return "".join(
-                (
-                    self.linebreak() * 2,
-                    text,
-                )
-            )
-        return ""
-
-    def block_quote(self, text):
-        return text
-
-    def link(self, link, title, content):
-        return "".join(
-            (
-                content,
-                " ({})".format(title) if title else "",
-                ": ",
-                link,
-            )
-        )
-
-    def autolink(self, link, is_email=False):  # noqa
-        return link
-
-
-class NotifyEmailPreheaderMarkdownRenderer(NotifyPlainTextEmailMarkdownRenderer):
-    def header(self, text, level, raw=None):  # noqa
-        return self.paragraph(text)
-
-    def hrule(self):
-        return ""
-
-    def link(self, link, title, content):
-        return "".join(
-            (
-                content,
-                " ({})".format(title) if title else "",
-            )
-        )
-
-
-notify_email_markdown = mistune.Markdown(
-    renderer=NotifyEmailMarkdownRenderer(),
-    hard_wrap=True,
-    use_xhtml=False,
 )
-notify_plain_text_email_markdown = mistune.Markdown(
+notify_letter_preview_markdown = mistune.create_markdown(
-    renderer=NotifyPlainTextEmailMarkdownRenderer(),
+    renderer=LetterPreviewRenderer()
-    hard_wrap=True,
 )
-notify_email_preheader_markdown = mistune.Markdown(
+notify_email_preheader_markdown = mistune.create_markdown(renderer=PreheaderRenderer())
-    renderer=NotifyEmailPreheaderMarkdownRenderer(),
+_notify_plain_text_email_markdown = mistune.create_markdown(
-    hard_wrap=True,
+    renderer=PlainTextRenderer()
-)
-notify_letter_preview_markdown = mistune.Markdown(
-    renderer=NotifyLetterMarkdownPreviewRenderer(),
-    hard_wrap=True,
-    use_xhtml=False,
 )
+
+
+def notify_email_markdown(text):
+    text = escape_plus_lists(text)
+    return _notify_email_markdown(autolinkify(text))
+
+
+def notify_plain_text_email_markdown(text):
+    text = escape_plus_lists(text)
+    return _notify_plain_text_email_markdown(text)