bugdown: add some type annotations

Had to add some "type: ignore" because the pattern used in match
doesn't affect the type returned. A fix for this issue has been pushed
to typeshed - python/typeshed#244

Author: Oren Leaffer

zerver/lib/bugdown/__init__.py

@@ -1,10 +1,11 @@
 from __future__ import absolute_import
 # Zulip's main markdown implementation.  See docs/markdown.md for
 # detailed documentation on our markdown syntax.
-from typing import Any, Optional
+from typing import Any, Optional, Callable, Union, Dict
 from typing.re import Match
 
 import markdown
+from markdown.util.etree import ElementTree
 import logging
 import traceback
 from six.moves import urllib
@@ -38,13 +39,15 @@
 import zerver.lib.mention as mention
 import six
 from six.moves import range
+from six import text_type
 
 
 # Format version of the bugdown rendering; stored along with rendered
 # messages so that we can efficiently determine what needs to be re-rendered
 version = 1
 
 def list_of_tlds():
+    # type: () -> List[str]
     # HACK we manually blacklist .py
     blacklist = ['PY\n', ]
 
@@ -56,6 +59,7 @@ def list_of_tlds():
     return tlds
 
 def walk_tree(root, processor, stop_after_first=False):
+    # type: (ElementTree, Callable[[ElementTree], Optional[ElementTree]], bool) -> List[ElementTree]
     results = []
     stack = [root]
 
@@ -76,6 +80,8 @@ def walk_tree(root, processor, stop_after_first=False):
 # height is not actually used
 def add_a(root, url, link, height="", title=None, desc=None,
           class_attr="message_inline_image"):
+    # type: ignore # (ElementTree, text_type, text_type, text_type, Optional[text_type], Optional[text_type], text_type) -> None
+    # above ignored until mypy picks up https://github.com/python/typeshed/pull/244
     title = title if title is not None else url_filename(link)
     title = title if title else ""
     desc = desc if desc is not None else ""
@@ -98,6 +104,7 @@ def add_a(root, url, link, height="", title=None, desc=None,
 
 @cache_with_key(lambda tweet_id: tweet_id, cache_name="database", with_statsd_key="tweet_data")
 def fetch_tweet_data(tweet_id):
+    # type: (text_type) -> Optional[Dict[text_type, text_type]]
     if settings.TEST_SUITE:
         from . import testing_mocks
         res = testing_mocks.twitter(tweet_id)
@@ -158,6 +165,7 @@ def fetch_tweet_data(tweet_id):
 META_END_RE = re.compile('^/meta[ >]')
 
 def fetch_open_graph_image(url):
+    # type: (str) -> Optional[Dict[str, Any]]
     in_head = False
     # HTML will auto close meta tags, when we start the next tag add a closing tag if it has not been closed yet.
     last_closed = True
@@ -223,21 +231,23 @@ def fetch_open_graph_image(url):
     return {'image': image, 'title': title, 'desc': desc}
 
 def get_tweet_id(url):
+    # type: (str) -> Union[bool, str]
     parsed_url = urllib.parse.urlparse(url)
     if not (parsed_url.netloc == 'twitter.com' or parsed_url.netloc.endswith('.twitter.com')):
-        return False
+        return False # TODO: probably should return None instead and change return type to Optional[str]
     to_match = parsed_url.path
     # In old-style twitter.com/#!/wdaher/status/1231241234-style URLs, we need to look at the fragment instead
     if parsed_url.path == '/' and len(parsed_url.fragment) > 5:
         to_match= parsed_url.fragment
 
     tweet_id_match = re.match(r'^!?/.*?/status(es)?/(?P<tweetid>\d{10,18})(/photo/[0-9])?/?$', to_match)
     if not tweet_id_match:
-        return False
+        return False # TODO: probably should return None instead and change return type to Optional[str]
     return tweet_id_match.group("tweetid")
 
 class InlineHttpsProcessor(markdown.treeprocessors.Treeprocessor):
     def run(self, root):
+        # type: (ElementTree) -> None
         # Get all URLs from the blob
         found_imgs = walk_tree(root, lambda e: e if e.tag == "img" else None)
         for img in found_imgs:
@@ -252,11 +262,13 @@ class InlineInterestingLinkProcessor(markdown.treeprocessors.Treeprocessor):
     TWITTER_MAX_TO_PREVIEW = 3
 
     def __init__(self, md, bugdown):
+        # type: (markdown.Markdown, Bugdown) -> None
         # Passing in bugdown for access to config to check if realm is zulip.com
         self.bugdown = bugdown
         markdown.treeprocessors.Treeprocessor.__init__(self, md)
 
     def is_image(self, url):
+        # type: (str) -> bool
         if not settings.INLINE_IMAGE_PREVIEW:
             return False
         parsed_url = urllib.parse.urlparse(url)
@@ -267,6 +279,8 @@ def is_image(self, url):
         return False
 
     def dropbox_image(self, url):
+        # type: (str) -> Optional[Dict]
+        # TODO: specify details of returned Dict
         parsed_url = urllib.parse.urlparse(url)
         if (parsed_url.netloc == 'dropbox.com' or parsed_url.netloc.endswith('.dropbox.com')):
             is_album = parsed_url.path.startswith('/sc/') or parsed_url.path.startswith('/photos/')
@@ -311,6 +325,7 @@ def dropbox_image(self, url):
         return None
 
     def youtube_image(self, url):
+        # type: (str) -> Optional[str]
         if not settings.INLINE_IMAGE_PREVIEW:
             return None
         # Youtube video id extraction regular expression from http://pastebin.com/KyKAFv1s
@@ -404,6 +419,7 @@ def set_text(text):
         return p
 
     def twitter_link(self, url):
+        # type: (str) -> Optional[markdown.util.etree.Element]
         tweet_id = get_tweet_id(url)
 
         if not tweet_id:
@@ -475,6 +491,7 @@ def twitter_link(self, url):
             return None
 
     def run(self, root):
+        # type: (ElementTree) -> None
         # Get all URLs from the blob
         found_urls = walk_tree(root, lambda e: e.get("href") if e.tag == "a" else None)
 
@@ -520,6 +537,7 @@ def run(self, root):
 
 class Avatar(markdown.inlinepatterns.Pattern):
     def handleMatch(self, match):
+        # type: (Match) -> markdown.util.etree.Element
         img = markdown.util.etree.Element('img')
         email_address = match.group('email')
         img.set('class', 'message_body_gravatar')
@@ -540,6 +558,7 @@ def handleMatch(self, match):
 
 
 def make_emoji(emoji_name, src, display_string):
+    # type: (str, str, str) -> markdown.util.etree.Element
     elt = markdown.util.etree.Element('img')
     elt.set('src', src)
     elt.set('class', 'emoji')
@@ -569,6 +588,7 @@ class StreamSubscribeButton(markdown.inlinepatterns.Pattern):
     # This markdown extension has required javascript in
     # static/js/custom_markdown.js
     def handleMatch(self, match):
+        # type: (Match) -> markdown.util.etree.Element
         stream_name = match.group('stream_name')
         stream_name = stream_name.replace('\\)', ')').replace('\\\\', '\\')
 
@@ -590,6 +610,7 @@ class ModalLink(markdown.inlinepatterns.Pattern):
     A pattern that allows including in-app modal links in messages.
     """
     def handleMatch(self, match):
+        # type: (Match) -> markdown.util.etree.Element
         relative_url = match.group('relative_url')
         text = match.group('text')
 
@@ -603,6 +624,8 @@ def handleMatch(self, match):
 
 upload_re = re.compile(r"^(?:https://%s.s3.amazonaws.com|/user_uploads/\d+)/[^/]*/([^/]*)$" % (settings.S3_BUCKET,))
 def url_filename(url):
+    # type: ignore # (text_type) -> text_type
+    # above ignored until mypy picks up https://github.com/python/typeshed/pull/244
     """Extract the filename if a URL is an uploaded file, or return the original URL"""
     match = upload_re.match(url)
     if match:
@@ -611,13 +634,15 @@ def url_filename(url):
         return url
 
 def fixup_link(link, target_blank=True):
+    # type: (markdown.element.etree.Element, bool) -> None
     """Set certain attributes we want on every link."""
     if target_blank:
         link.set('target', '_blank')
     link.set('title',  url_filename(link.get('href')))
 
 
 def sanitize_url(url):
+    # type: (text_type) -> text_type
     """
     Sanitize a url against xss attacks.
     See the docstring on markdown.inlinepatterns.LinkPattern.sanitize_url.