Merge pull request #270 from gsnedders/rename_stuff

Author: gsnedders

CHANGES.rst

@@ -50,6 +50,20 @@ Released on XXX
   with a set of keyword arguments: override_encoding, transport_encoding,
   same_origin_parent_encoding, likely_encoding, and default_encoding.**
 
+* **Move filters._base, treebuilder._base, and treewalkers._base to .base
+  to clarify their status as public.**
+
+* **Get rid of the sanitizer package. Merge sanitizer.sanitize into the
+  sanitizer.htmlsanitizer module and move that to saniziter. This means
+  anyone who used sanitizer.sanitize or sanitizer.HTMLSanitizer needs no
+  code changes.**
+
+* **Rename treewalkers.lxmletree to .etree_lxml and
+  treewalkers.genshistream to .genshi to have a consistent API.**
+
+* Move a whole load of stuff (inputstream, ihatexml, trie, tokenizer,
+  utils) to be underscore prefixed to clarify their status as private.
+
 
 0.9999999/1.0b8
 ~~~~~~~~~~~~~~~

doc/html5lib.filters.rst

@@ -1,10 +1,10 @@
 filters Package
 ===============
 
-:mod:`_base` Module
+:mod:`base` Module
 -------------------
 
-.. automodule:: html5lib.filters._base
+.. automodule:: html5lib.filters.base
     :members:
     :undoc-members:
     :show-inheritance:

doc/html5lib.rst

@@ -25,42 +25,10 @@ html5lib Package
     :undoc-members:
     :show-inheritance:
 
-:mod:`ihatexml` Module
+:mod:`serializer` Module
 ----------------------
 
-.. automodule:: html5lib.ihatexml
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-:mod:`inputstream` Module
--------------------------
-
-.. automodule:: html5lib.inputstream
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-:mod:`sanitizer` Module
------------------------
-
-.. automodule:: html5lib.sanitizer
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-:mod:`tokenizer` Module
------------------------
-
-.. automodule:: html5lib.tokenizer
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-:mod:`utils` Module
--------------------
-
-.. automodule:: html5lib.utils
+.. automodule:: html5lib.serializer
     :members:
     :undoc-members:
     :show-inheritance:
@@ -71,7 +39,6 @@ Subpackages
 .. toctree::
 
     html5lib.filters
-    html5lib.serializer
     html5lib.treebuilders
     html5lib.treewalkers
 

doc/html5lib.serializer.rst

@@ -9,10 +9,10 @@ treebuilders Package
     :undoc-members:
     :show-inheritance:
 
-:mod:`_base` Module
+:mod:`base` Module
 -------------------
 
-.. automodule:: html5lib.treebuilders._base
+.. automodule:: html5lib.treebuilders.base
     :members:
     :undoc-members:
     :show-inheritance:

doc/html5lib.treebuilders.rst

@@ -9,10 +9,10 @@ treewalkers Package
     :undoc-members:
     :show-inheritance:
 
-:mod:`_base` Module
+:mod:`base` Module
 -------------------
 
-.. automodule:: html5lib.treewalkers._base
+.. automodule:: html5lib.treewalkers.base
     :members:
     :undoc-members:
     :show-inheritance:
@@ -33,18 +33,19 @@ treewalkers Package
     :undoc-members:
     :show-inheritance:
 
-:mod:`genshistream` Module
---------------------------
+:mod:`etree_lxml` Module
+-----------------------
 
-.. automodule:: html5lib.treewalkers.genshistream
+.. automodule:: html5lib.treewalkers.etree_lxml
     :members:
     :undoc-members:
     :show-inheritance:
 
-:mod:`lxmletree` Module
------------------------
 
-.. automodule:: html5lib.treewalkers.lxmletree
+:mod:`genshi` Module
+--------------------------
+
+.. automodule:: html5lib.treewalkers.genshi
     :members:
     :undoc-members:
-    :show-inheritance:
+    :show-inheritance:

doc/html5lib.treewalkers.rst

@@ -10,7 +10,7 @@
 
 from .constants import EOF, spaceCharacters, asciiLetters, asciiUppercase
 from .constants import ReparseException
-from . import utils
+from . import _utils
 
 from io import StringIO
 
@@ -28,7 +28,7 @@
 
 invalid_unicode_no_surrogate = "[\u0001-\u0008\u000B\u000E-\u001F\u007F-\u009F\uFDD0-\uFDEF\uFFFE\uFFFF\U0001FFFE\U0001FFFF\U0002FFFE\U0002FFFF\U0003FFFE\U0003FFFF\U0004FFFE\U0004FFFF\U0005FFFE\U0005FFFF\U0006FFFE\U0006FFFF\U0007FFFE\U0007FFFF\U0008FFFE\U0008FFFF\U0009FFFE\U0009FFFF\U000AFFFE\U000AFFFF\U000BFFFE\U000BFFFF\U000CFFFE\U000CFFFF\U000DFFFE\U000DFFFF\U000EFFFE\U000EFFFF\U000FFFFE\U000FFFFF\U0010FFFE\U0010FFFF]"  # noqa
 
-if utils.supports_lone_surrogates:
+if _utils.supports_lone_surrogates:
     # Use one extra step of indirection and create surrogates with
     # eval. Not using this indirection would introduce an illegal
     # unicode literal on platforms not supporting such lone
@@ -176,7 +176,7 @@ def __init__(self, source):
 
         """
 
-        if not utils.supports_lone_surrogates:
+        if not _utils.supports_lone_surrogates:
             # Such platforms will have already checked for such
             # surrogate errors, so no need to do this checking.
             self.reportCharacterErrors = None
@@ -304,9 +304,9 @@ def characterErrorsUCS2(self, data):
             codepoint = ord(match.group())
             pos = match.start()
             # Pretty sure there should be endianness issues here
-            if utils.isSurrogatePair(data[pos:pos + 2]):
+            if _utils.isSurrogatePair(data[pos:pos + 2]):
                 # We have a surrogate pair!
-                char_val = utils.surrogatePairToCodepoint(data[pos:pos + 2])
+                char_val = _utils.surrogatePairToCodepoint(data[pos:pos + 2])
                 if char_val in non_bmp_invalid_codepoints:
                     self.errors.append("invalid-codepoint")
                 skip = True

html5lib/ihatexml.py renamed to html5lib/_ihatexml.py

@@ -11,9 +11,9 @@
 from .constants import tokenTypes, tagTokenTypes
 from .constants import replacementCharacters
 
-from .inputstream import HTMLInputStream
+from ._inputstream import HTMLInputStream
 
-from .trie import Trie
+from ._trie import Trie
 
 entitiesTrie = Trie(entities)
 

html5lib/inputstream.py renamed to html5lib/_inputstream.py

@@ -1,16 +1,16 @@
 from __future__ import absolute_import, division, unicode_literals
 
-from . import _base
+from . import base
 
 try:
     from collections import OrderedDict
 except ImportError:
     from ordereddict import OrderedDict
 
 
-class Filter(_base.Filter):
+class Filter(base.Filter):
     def __iter__(self):
-        for token in _base.Filter.__iter__(self):
+        for token in base.Filter.__iter__(self):
             if token["type"] in ("StartTag", "EmptyTag"):
                 attrs = OrderedDict()
                 for name, value in sorted(token["data"].items(),

html5lib/tokenizer.py renamed to html5lib/_tokenizer.py

@@ -1,19 +1,19 @@
 from __future__ import absolute_import, division, unicode_literals
 
-from . import _base
+from . import base
 
 
-class Filter(_base.Filter):
+class Filter(base.Filter):
     def __init__(self, source, encoding):
-        _base.Filter.__init__(self, source)
+        base.Filter.__init__(self, source)
         self.encoding = encoding
 
     def __iter__(self):
         state = "pre_head"
         meta_found = (self.encoding is None)
         pending = []
 
-        for token in _base.Filter.__iter__(self):
+        for token in base.Filter.__iter__(self):
             type = token["type"]
             if type == "StartTag":
                 if token["name"].lower() == "head":

html5lib/trie/__init__.py renamed to html5lib/_trie/__init__.py

@@ -2,21 +2,21 @@
 
 from six import text_type
 
-from . import _base
+from . import base
 from ..constants import namespaces, voidElements
 
 from ..constants import spaceCharacters
 spaceCharacters = "".join(spaceCharacters)
 
 
-class Filter(_base.Filter):
+class Filter(base.Filter):
     def __init__(self, source, require_matching_tags=True):
         super(Filter, self).__init__(source)
         self.require_matching_tags = require_matching_tags
 
     def __iter__(self):
         open_elements = []
-        for token in _base.Filter.__iter__(self):
+        for token in base.Filter.__iter__(self):
             type = token["type"]
             if type in ("StartTag", "EmptyTag"):
                 namespace = token["namespace"]

html5lib/trie/_base.py renamed to html5lib/_trie/_base.py

@@ -1,9 +1,9 @@
 from __future__ import absolute_import, division, unicode_literals
 
-from . import _base
+from . import base
 
 
-class Filter(_base.Filter):
+class Filter(base.Filter):
     def slider(self):
         previous1 = previous2 = None
         for token in self.source:

html5lib/trie/datrie.py renamed to html5lib/_trie/datrie.py

@@ -5,13 +5,13 @@
 
 from six.moves import urllib_parse as urlparse
 
-from . import _base
+from . import base
 from ..constants import namespaces, prefixes
 
 __all__ = ["Filter"]
 
 
-acceptable_elements = frozenset((
+allowed_elements = frozenset((
     (namespaces['html'], 'a'),
     (namespaces['html'], 'abbr'),
     (namespaces['html'], 'acronym'),
@@ -175,7 +175,7 @@
     (namespaces['svg'], 'use'),
 ))
 
-acceptable_attributes = frozenset((
+allowed_attributes = frozenset((
     # HTML attributes
     (None, 'abbr'),
     (None, 'accept'),
@@ -552,7 +552,7 @@
     (None, 'use')
 ))
 
-acceptable_css_properties = frozenset((
+allowed_css_properties = frozenset((
     'azimuth',
     'background-color',
     'border-bottom-color',
@@ -601,7 +601,7 @@
     'width',
 ))
 
-acceptable_css_keywords = frozenset((
+allowed_css_keywords = frozenset((
     'auto',
     'aqua',
     'black',
@@ -643,7 +643,7 @@
     'yellow',
 ))
 
-acceptable_svg_properties = frozenset((
+allowed_svg_properties = frozenset((
     'fill',
     'fill-opacity',
     'fill-rule',
@@ -654,7 +654,7 @@
     'stroke-opacity',
 ))
 
-acceptable_protocols = frozenset((
+allowed_protocols = frozenset((
     'ed2k',
     'ftp',
     'http',
@@ -680,7 +680,7 @@
     'data',
 ))
 
-acceptable_content_types = frozenset((
+allowed_content_types = frozenset((
     'image/png',
     'image/jpeg',
     'image/gif',
@@ -689,14 +689,6 @@
     'text/plain',
 ))
 
-allowed_elements = acceptable_elements
-allowed_attributes = acceptable_attributes
-allowed_css_properties = acceptable_css_properties
-allowed_css_keywords = acceptable_css_keywords
-allowed_svg_properties = acceptable_svg_properties
-allowed_protocols = acceptable_protocols
-allowed_content_types = acceptable_content_types
-
 
 data_content_type = re.compile(r'''
                                 ^
@@ -712,7 +704,7 @@
                                re.VERBOSE)
 
 
-class Filter(_base.Filter):
+class Filter(base.Filter):
     """ sanitization of XHTML+MathML+SVG and of inline style attributes."""
     def __init__(self,
                  source,
@@ -739,7 +731,7 @@ def __init__(self,
         self.svg_allow_local_href = svg_allow_local_href
 
     def __iter__(self):
-        for token in _base.Filter.__iter__(self):
+        for token in base.Filter.__iter__(self):
             token = self.sanitize_token(token)
             if token:
                 yield token

html5lib/trie/py.py renamed to html5lib/_trie/py.py

@@ -2,20 +2,20 @@
 
 import re
 
-from . import _base
+from . import base
 from ..constants import rcdataElements, spaceCharacters
 spaceCharacters = "".join(spaceCharacters)
 
 SPACES_REGEX = re.compile("[%s]+" % spaceCharacters)
 
 
-class Filter(_base.Filter):
+class Filter(base.Filter):
 
     spacePreserveElements = frozenset(["pre", "textarea"] + list(rcdataElements))
 
     def __iter__(self):
         preserve = 0
-        for token in _base.Filter.__iter__(self):
+        for token in base.Filter.__iter__(self):
             type = token["type"]
             if type == "StartTag" \
                     and (preserve or token["name"] in self.spacePreserveElements):

html5lib/utils.py renamed to html5lib/_utils.py

@@ -5,40 +5,38 @@
 
 from codecs import register_error, xmlcharrefreplace_errors
 
-from ..constants import voidElements, booleanAttributes, spaceCharacters
-from ..constants import rcdataElements, entities, xmlEntities
-from .. import utils
+from .constants import voidElements, booleanAttributes, spaceCharacters
+from .constants import rcdataElements, entities, xmlEntities
+from . import treewalkers, _utils
 from xml.sax.saxutils import escape
 
-spaceCharacters = "".join(spaceCharacters)
+_quoteAttributeSpecChars = "".join(spaceCharacters) + "\"'=<>`"
+_quoteAttributeSpec = re.compile("[" + _quoteAttributeSpecChars + "]")
+_quoteAttributeLegacy = re.compile("[" + _quoteAttributeSpecChars +
+                                   "\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n"
+                                   "\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15"
+                                   "\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
+                                   "\x20\x2f\x60\xa0\u1680\u180e\u180f\u2000"
+                                   "\u2001\u2002\u2003\u2004\u2005\u2006\u2007"
+                                   "\u2008\u2009\u200a\u2028\u2029\u202f\u205f"
+                                   "\u3000]")
 
-quoteAttributeSpecChars = spaceCharacters + "\"'=<>`"
-quoteAttributeSpec = re.compile("[" + quoteAttributeSpecChars + "]")
-quoteAttributeLegacy = re.compile("[" + quoteAttributeSpecChars +
-                                  "\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n"
-                                  "\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15"
-                                  "\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
-                                  "\x20\x2f\x60\xa0\u1680\u180e\u180f\u2000"
-                                  "\u2001\u2002\u2003\u2004\u2005\u2006\u2007"
-                                  "\u2008\u2009\u200a\u2028\u2029\u202f\u205f"
-                                  "\u3000]")
 
-
-encode_entity_map = {}
-is_ucs4 = len("\U0010FFFF") == 1
+_encode_entity_map = {}
+_is_ucs4 = len("\U0010FFFF") == 1
 for k, v in list(entities.items()):
     # skip multi-character entities
-    if ((is_ucs4 and len(v) > 1) or
-            (not is_ucs4 and len(v) > 2)):
+    if ((_is_ucs4 and len(v) > 1) or
+            (not _is_ucs4 and len(v) > 2)):
         continue
     if v != "&":
         if len(v) == 2:
-            v = utils.surrogatePairToCodepoint(v)
+            v = _utils.surrogatePairToCodepoint(v)
         else:
             v = ord(v)
-        if v not in encode_entity_map or k.islower():
+        if v not in _encode_entity_map or k.islower():
             # prefer &lt; over &LT; and similarly for &amp;, &gt;, etc.
-            encode_entity_map[v] = k
+            _encode_entity_map[v] = k
 
 
 def htmlentityreplace_errors(exc):
@@ -51,14 +49,14 @@ def htmlentityreplace_errors(exc):
                 skip = False
                 continue
             index = i + exc.start
-            if utils.isSurrogatePair(exc.object[index:min([exc.end, index + 2])]):
-                codepoint = utils.surrogatePairToCodepoint(exc.object[index:index + 2])
+            if _utils.isSurrogatePair(exc.object[index:min([exc.end, index + 2])]):
+                codepoint = _utils.surrogatePairToCodepoint(exc.object[index:index + 2])
                 skip = True
             else:
                 codepoint = ord(c)
             codepoints.append(codepoint)
         for cp in codepoints:
-            e = encode_entity_map.get(cp)
+            e = _encode_entity_map.get(cp)
             if e:
                 res.append("&")
                 res.append(e)
@@ -73,6 +71,13 @@ def htmlentityreplace_errors(exc):
 register_error("htmlentityreplace", htmlentityreplace_errors)
 
 
+def serialize(input, tree="etree", encoding=None, **serializer_opts):
+    # XXX: Should we cache this?
+    walker = treewalkers.getTreeWalker(tree)
+    s = HTMLSerializer(**serializer_opts)
+    return s.render(walker(input), encoding)
+
+
 class HTMLSerializer(object):
 
     # attribute quoting options
@@ -181,24 +186,24 @@ def serialize(self, treewalker, encoding=None):
         self.errors = []
 
         if encoding and self.inject_meta_charset:
-            from ..filters.inject_meta_charset import Filter
+            from .filters.inject_meta_charset import Filter
             treewalker = Filter(treewalker, encoding)
         # Alphabetical attributes is here under the assumption that none of
         # the later filters add or change order of attributes; it needs to be
         # before the sanitizer so escaped elements come out correctly
         if self.alphabetical_attributes:
-            from ..filters.alphabeticalattributes import Filter
+            from .filters.alphabeticalattributes import Filter
             treewalker = Filter(treewalker)
         # WhitespaceFilter should be used before OptionalTagFilter
         # for maximum efficiently of this latter filter
         if self.strip_whitespace:
-            from ..filters.whitespace import Filter
+            from .filters.whitespace import Filter
             treewalker = Filter(treewalker)
         if self.sanitize:
-            from ..filters.sanitizer import Filter
+            from .filters.sanitizer import Filter
             treewalker = Filter(treewalker)
         if self.omit_optional_tags:
-            from ..filters.optionaltags import Filter
+            from .filters.optionaltags import Filter
             treewalker = Filter(treewalker)
 
         for token in treewalker:
@@ -251,9 +256,9 @@ def serialize(self, treewalker, encoding=None):
                         if self.quote_attr_values == "always" or len(v) == 0:
                             quote_attr = True
                         elif self.quote_attr_values == "spec":
-                            quote_attr = quoteAttributeSpec.search(v) is not None
+                            quote_attr = _quoteAttributeSpec.search(v) is not None
                         elif self.quote_attr_values == "legacy":
-                            quote_attr = quoteAttributeLegacy.search(v) is not None
+                            quote_attr = _quoteAttributeLegacy.search(v) is not None
                         else:
                             raise ValueError("quote_attr_values must be one of: "
                                              "'always', 'spec', or 'legacy'")

html5lib/filters/alphabeticalattributes.py

@@ -5,15 +5,15 @@
 import pytest
 
 from .support import get_data_files, test_dir, errorMessage, TestData as _TestData
-from html5lib import HTMLParser, inputstream
+from html5lib import HTMLParser, _inputstream
 
 
 def test_basic_prescan_length():
     data = "<title>Caf\u00E9</title><!--a--><meta charset='utf-8'>".encode('utf-8')
     pad = 1024 - len(data) + 1
     data = data.replace(b"-a-", b"-" + (b"a" * pad) + b"-")
     assert len(data) == 1024  # Sanity
-    stream = inputstream.HTMLBinaryInputStream(data, useChardet=False)
+    stream = _inputstream.HTMLBinaryInputStream(data, useChardet=False)
     assert 'utf-8' == stream.charEncoding[0].name
 
 
@@ -22,7 +22,7 @@ def test_parser_reparse():
     pad = 10240 - len(data) + 1
     data = data.replace(b"-a-", b"-" + (b"a" * pad) + b"-")
     assert len(data) == 10240  # Sanity
-    stream = inputstream.HTMLBinaryInputStream(data, useChardet=False)
+    stream = _inputstream.HTMLBinaryInputStream(data, useChardet=False)
     assert 'windows-1252' == stream.charEncoding[0].name
     p = HTMLParser(namespaceHTMLElements=False)
     doc = p.parse(data, useChardet=False)
@@ -47,7 +47,7 @@ def test_parser_reparse():
     ("windows-1252", b"", {}),
 ])
 def test_parser_args(expected, data, kwargs):
-    stream = inputstream.HTMLBinaryInputStream(data, useChardet=False, **kwargs)
+    stream = _inputstream.HTMLBinaryInputStream(data, useChardet=False, **kwargs)
     assert expected == stream.charEncoding[0].name
     p = HTMLParser()
     p.parse(data, useChardet=False, **kwargs)
@@ -85,7 +85,7 @@ def runParserEncodingTest(data, encoding):
 
 
 def runPreScanEncodingTest(data, encoding):
-    stream = inputstream.HTMLBinaryInputStream(data, useChardet=False)
+    stream = _inputstream.HTMLBinaryInputStream(data, useChardet=False)
     encoding = encoding.lower().decode("ascii")
 
     # Very crude way to ignore irrelevant tests
@@ -111,6 +111,6 @@ def test_encoding():
 else:
     def test_chardet():
         with open(os.path.join(test_dir, "encoding", "chardet", "test_big5.txt"), "rb") as fp:
-            encoding = inputstream.HTMLInputStream(fp.read()).charEncoding
+            encoding = _inputstream.HTMLInputStream(fp.read()).charEncoding
             assert encoding[0].name == "big5"
 # pylint:enable=wrong-import-position

html5lib/filters/_base.py renamed to html5lib/filters/base.py

@@ -10,7 +10,7 @@
 from html5lib import constants
 from html5lib.filters.lint import Filter as Lint
 from html5lib.serializer import HTMLSerializer, serialize
-from html5lib.treewalkers._base import TreeWalker
+from html5lib.treewalkers.base import TreeWalker
 
 # pylint:disable=wrong-import-position
 optionals_loaded = []

html5lib/filters/inject_meta_charset.py

@@ -11,9 +11,9 @@
 import six
 from six.moves import http_client, urllib
 
-from html5lib.inputstream import (BufferedStream, HTMLInputStream,
-                                  HTMLUnicodeInputStream, HTMLBinaryInputStream)
-from html5lib.utils import supports_lone_surrogates
+from html5lib._inputstream import (BufferedStream, HTMLInputStream,
+                                   HTMLUnicodeInputStream, HTMLBinaryInputStream)
+from html5lib._utils import supports_lone_surrogates
 
 
 def test_basic():

html5lib/filters/lint.py

@@ -8,8 +8,8 @@
 import pytest
 from six import unichr
 
-from html5lib.tokenizer import HTMLTokenizer
-from html5lib import constants, utils
+from html5lib._tokenizer import HTMLTokenizer
+from html5lib import constants, _utils
 
 
 class TokenizerTestParser(object):
@@ -156,7 +156,7 @@ def repl(m):
         except ValueError:
             # This occurs when unichr throws ValueError, which should
             # only be for a lone-surrogate.
-            if utils.supports_lone_surrogates:
+            if _utils.supports_lone_surrogates:
                 raise
             return None
 

html5lib/filters/optionaltags.py

@@ -28,7 +28,7 @@
 
 from __future__ import absolute_import, division, unicode_literals
 
-from ..utils import default_etree
+from .._utils import default_etree
 
 treeBuilderCache = {}
 

html5lib/filters/sanitizer.py

@@ -5,10 +5,10 @@
 from xml.dom import minidom, Node
 import weakref
 
-from . import _base
+from . import base
 from .. import constants
 from ..constants import namespaces
-from ..utils import moduleFactoryFactory
+from .._utils import moduleFactoryFactory
 
 
 def getDomBuilder(DomImplementation):
@@ -50,9 +50,9 @@ def __delitem__(self, name):
             else:
                 del self.element.attributes[name]
 
-    class NodeBuilder(_base.Node):
+    class NodeBuilder(base.Node):
         def __init__(self, element):
-            _base.Node.__init__(self, element.nodeName)
+            base.Node.__init__(self, element.nodeName)
             self.element = element
 
         namespace = property(lambda self: hasattr(self.element, "namespaceURI") and
@@ -117,7 +117,7 @@ def getNameTuple(self):
 
         nameTuple = property(getNameTuple)
 
-    class TreeBuilder(_base.TreeBuilder):  # pylint:disable=unused-variable
+    class TreeBuilder(base.TreeBuilder):  # pylint:disable=unused-variable
         def documentClass(self):
             self.dom = Dom.getDOMImplementation().createDocument(None, None, None)
             return weakref.proxy(self)
@@ -157,12 +157,12 @@ def getDocument(self):
             return self.dom
 
         def getFragment(self):
-            return _base.TreeBuilder.getFragment(self).element
+            return base.TreeBuilder.getFragment(self).element
 
         def insertText(self, data, parent=None):
             data = data
             if parent != self:
-                _base.TreeBuilder.insertText(self, data, parent)
+                base.TreeBuilder.insertText(self, data, parent)
             else:
                 # HACK: allow text nodes as children of the document node
                 if hasattr(self.dom, '_child_node_types'):

html5lib/filters/whitespace.py

@@ -5,11 +5,11 @@
 
 import re
 
-from . import _base
-from .. import ihatexml
+from . import base
+from .. import _ihatexml
 from .. import constants
 from ..constants import namespaces
-from ..utils import moduleFactoryFactory
+from .._utils import moduleFactoryFactory
 
 tag_regexp = re.compile("{([^}]*)}(.*)")
 
@@ -18,7 +18,7 @@ def getETreeBuilder(ElementTreeImplementation, fullTree=False):
     ElementTree = ElementTreeImplementation
     ElementTreeCommentType = ElementTree.Comment("asd").tag
 
-    class Element(_base.Node):
+    class Element(base.Node):
         def __init__(self, name, namespace=None):
             self._name = name
             self._namespace = namespace
@@ -142,7 +142,7 @@ def reparentChildren(self, newParent):
                 if self._element.text is not None:
                     newParent._element.text += self._element.text
             self._element.text = ""
-            _base.Node.reparentChildren(self, newParent)
+            base.Node.reparentChildren(self, newParent)
 
     class Comment(Element):
         def __init__(self, data):
@@ -259,7 +259,7 @@ def serializeElement(element, indent=0):
     def tostring(element):  # pylint:disable=unused-variable
         """Serialize an element and its child nodes to a string"""
         rv = []
-        filter = ihatexml.InfosetFilter()
+        filter = _ihatexml.InfosetFilter()
 
         def serializeElement(element):
             if isinstance(element, ElementTree.ElementTree):
@@ -310,7 +310,7 @@ def serializeElement(element):
 
         return "".join(rv)
 
-    class TreeBuilder(_base.TreeBuilder):  # pylint:disable=unused-variable
+    class TreeBuilder(base.TreeBuilder):  # pylint:disable=unused-variable
         documentClass = Document
         doctypeClass = DocumentType
         elementClass = Element
@@ -332,7 +332,7 @@ def getDocument(self):
                     return self.document._element.find("html")
 
         def getFragment(self):
-            return _base.TreeBuilder.getFragment(self)._element
+            return base.TreeBuilder.getFragment(self)._element
 
     return locals()
 

html5lib/html5parser.py

@@ -16,11 +16,11 @@
 import re
 import sys
 
-from . import _base
+from . import base
 from ..constants import DataLossWarning
 from .. import constants
 from . import etree as etree_builders
-from .. import ihatexml
+from .. import _ihatexml
 
 import lxml.etree as etree
 
@@ -54,7 +54,7 @@ def _getChildNodes(self):
 
 def testSerializer(element):
     rv = []
-    infosetFilter = ihatexml.InfosetFilter(preventDoubleDashComments=True)
+    infosetFilter = _ihatexml.InfosetFilter(preventDoubleDashComments=True)
 
     def serializeElement(element, indent=0):
         if not hasattr(element, "tag"):
@@ -172,7 +172,7 @@ def serializeElement(element):
     return "".join(rv)
 
 
-class TreeBuilder(_base.TreeBuilder):
+class TreeBuilder(base.TreeBuilder):
     documentClass = Document
     doctypeClass = DocumentType
     elementClass = None
@@ -182,7 +182,7 @@ class TreeBuilder(_base.TreeBuilder):
 
     def __init__(self, namespaceHTMLElements, fullTree=False):
         builder = etree_builders.getETreeModule(etree, fullTree=fullTree)
-        infosetFilter = self.infosetFilter = ihatexml.InfosetFilter(preventDoubleDashComments=True)
+        infosetFilter = self.infosetFilter = _ihatexml.InfosetFilter(preventDoubleDashComments=True)
         self.namespaceHTMLElements = namespaceHTMLElements
 
         class Attributes(dict):
@@ -254,10 +254,10 @@ def _getData(self):
         self.elementClass = Element
         self.commentClass = Comment
         # self.fragmentClass = builder.DocumentFragment
-        _base.TreeBuilder.__init__(self, namespaceHTMLElements)
+        base.TreeBuilder.__init__(self, namespaceHTMLElements)
 
     def reset(self):
-        _base.TreeBuilder.reset(self)
+        base.TreeBuilder.reset(self)
         self.insertComment = self.insertCommentInitial
         self.initial_comments = []
         self.doctype = None

html5lib/serializer/htmlserializer.py renamed to html5lib/serializer.py

@@ -11,9 +11,9 @@
 from __future__ import absolute_import, division, unicode_literals
 
 from .. import constants
-from ..utils import default_etree
+from .._utils import default_etree
 
-__all__ = ["getTreeWalker", "pprint", "dom", "etree", "genshistream", "lxmletree"]
+__all__ = ["getTreeWalker", "pprint", "dom", "etree", "genshi", "etree_lxml"]
 
 treeWalkerCache = {}
 
@@ -43,11 +43,11 @@ def getTreeWalker(treeType, implementation=None, **kwargs):
             from . import dom
             treeWalkerCache[treeType] = dom.TreeWalker
         elif treeType == "genshi":
-            from . import genshistream
-            treeWalkerCache[treeType] = genshistream.TreeWalker
+            from . import genshi
+            treeWalkerCache[treeType] = genshi.TreeWalker
         elif treeType == "lxml":
-            from . import lxmletree
-            treeWalkerCache[treeType] = lxmletree.TreeWalker
+            from . import etree_lxml
+            treeWalkerCache[treeType] = etree_lxml.TreeWalker
         elif treeType == "etree":
             from . import etree
             if implementation is None:

html5lib/serializer/__init__.py

@@ -2,16 +2,16 @@
 
 from xml.dom import Node
 
-from . import _base
+from . import base
 
 
-class TreeWalker(_base.NonRecursiveTreeWalker):
+class TreeWalker(base.NonRecursiveTreeWalker):
     def getNodeDetails(self, node):
         if node.nodeType == Node.DOCUMENT_TYPE_NODE:
-            return _base.DOCTYPE, node.name, node.publicId, node.systemId
+            return base.DOCTYPE, node.name, node.publicId, node.systemId
 
         elif node.nodeType in (Node.TEXT_NODE, Node.CDATA_SECTION_NODE):
-            return _base.TEXT, node.nodeValue
+            return base.TEXT, node.nodeValue
 
         elif node.nodeType == Node.ELEMENT_NODE:
             attrs = {}
@@ -21,17 +21,17 @@ def getNodeDetails(self, node):
                     attrs[(attr.namespaceURI, attr.localName)] = attr.value
                 else:
                     attrs[(None, attr.name)] = attr.value
-            return (_base.ELEMENT, node.namespaceURI, node.nodeName,
+            return (base.ELEMENT, node.namespaceURI, node.nodeName,
                     attrs, node.hasChildNodes())
 
         elif node.nodeType == Node.COMMENT_NODE:
-            return _base.COMMENT, node.nodeValue
+            return base.COMMENT, node.nodeValue
 
         elif node.nodeType in (Node.DOCUMENT_NODE, Node.DOCUMENT_FRAGMENT_NODE):
-            return (_base.DOCUMENT,)
+            return (base.DOCUMENT,)
 
         else:
-            return _base.UNKNOWN, node.nodeType
+            return base.UNKNOWN, node.nodeType
 
     def getFirstChild(self, node):
         return node.firstChild

html5lib/tests/test_encoding.py

@@ -12,8 +12,8 @@
 
 from six import string_types
 
-from . import _base
-from ..utils import moduleFactoryFactory
+from . import base
+from .._utils import moduleFactoryFactory
 
 tag_regexp = re.compile("{([^}]*)}(.*)")
 
@@ -22,7 +22,7 @@ def getETreeBuilder(ElementTreeImplementation):
     ElementTree = ElementTreeImplementation
     ElementTreeCommentType = ElementTree.Comment("asd").tag
 
-    class TreeWalker(_base.NonRecursiveTreeWalker):  # pylint:disable=unused-variable
+    class TreeWalker(base.NonRecursiveTreeWalker):  # pylint:disable=unused-variable
         """Given the particular ElementTree representation, this implementation,
         to avoid using recursion, returns "nodes" as tuples with the following
         content:
@@ -40,22 +40,22 @@ def getNodeDetails(self, node):
             if isinstance(node, tuple):  # It might be the root Element
                 elt, _, _, flag = node
                 if flag in ("text", "tail"):
-                    return _base.TEXT, getattr(elt, flag)
+                    return base.TEXT, getattr(elt, flag)
                 else:
                     node = elt
 
             if not(hasattr(node, "tag")):
                 node = node.getroot()
 
             if node.tag in ("DOCUMENT_ROOT", "DOCUMENT_FRAGMENT"):
-                return (_base.DOCUMENT,)
+                return (base.DOCUMENT,)
 
             elif node.tag == "<!DOCTYPE>":
-                return (_base.DOCTYPE, node.text,
+                return (base.DOCTYPE, node.text,
                         node.get("publicId"), node.get("systemId"))
 
             elif node.tag == ElementTreeCommentType:
-                return _base.COMMENT, node.text
+                return base.COMMENT, node.text
 
             else:
                 assert isinstance(node.tag, string_types), type(node.tag)
@@ -73,7 +73,7 @@ def getNodeDetails(self, node):
                         attrs[(match.group(1), match.group(2))] = value
                     else:
                         attrs[(None, name)] = value
-                return (_base.ELEMENT, namespace, tag,
+                return (base.ELEMENT, namespace, tag,
                         attrs, len(node) or node.text)
 
         def getFirstChild(self, node):

html5lib/tests/test_serializer.py

@@ -4,9 +4,9 @@
 from lxml import etree
 from ..treebuilders.etree import tag_regexp
 
-from . import _base
+from . import base
 
-from .. import ihatexml
+from .. import _ihatexml
 
 
 def ensure_str(s):
@@ -122,7 +122,7 @@ def __len__(self):
         return len(self.obj)
 
 
-class TreeWalker(_base.NonRecursiveTreeWalker):
+class TreeWalker(base.NonRecursiveTreeWalker):
     def __init__(self, tree):
         # pylint:disable=redefined-variable-type
         if isinstance(tree, list):
@@ -131,29 +131,29 @@ def __init__(self, tree):
         else:
             self.fragmentChildren = set()
             tree = Root(tree)
-        _base.NonRecursiveTreeWalker.__init__(self, tree)
-        self.filter = ihatexml.InfosetFilter()
+        base.NonRecursiveTreeWalker.__init__(self, tree)
+        self.filter = _ihatexml.InfosetFilter()
 
     def getNodeDetails(self, node):
         if isinstance(node, tuple):  # Text node
             node, key = node
             assert key in ("text", "tail"), "Text nodes are text or tail, found %s" % key
-            return _base.TEXT, ensure_str(getattr(node, key))
+            return base.TEXT, ensure_str(getattr(node, key))
 
         elif isinstance(node, Root):
-            return (_base.DOCUMENT,)
+            return (base.DOCUMENT,)
 
         elif isinstance(node, Doctype):
-            return _base.DOCTYPE, node.name, node.public_id, node.system_id
+            return base.DOCTYPE, node.name, node.public_id, node.system_id
 
         elif isinstance(node, FragmentWrapper) and not hasattr(node, "tag"):
-            return _base.TEXT, ensure_str(node.obj)
+            return base.TEXT, ensure_str(node.obj)
 
         elif node.tag == etree.Comment:
-            return _base.COMMENT, ensure_str(node.text)
+            return base.COMMENT, ensure_str(node.text)
 
         elif node.tag == etree.Entity:
-            return _base.ENTITY, ensure_str(node.text)[1:-1]  # strip &;
+            return base.ENTITY, ensure_str(node.text)[1:-1]  # strip &;
 
         else:
             # This is assumed to be an ordinary element
@@ -172,7 +172,7 @@ def getNodeDetails(self, node):
                     attrs[(match.group(1), match.group(2))] = value
                 else:
                     attrs[(None, name)] = value
-            return (_base.ELEMENT, namespace, self.filter.fromXmlName(tag),
+            return (base.ELEMENT, namespace, self.filter.fromXmlName(tag),
                     attrs, len(node) > 0 or node.text)
 
     def getFirstChild(self, node):

html5lib/tests/test_stream.py

@@ -4,12 +4,12 @@
 from genshi.core import START, END, XML_NAMESPACE, DOCTYPE, TEXT
 from genshi.core import START_NS, END_NS, START_CDATA, END_CDATA, PI, COMMENT
 
-from . import _base
+from . import base
 
 from ..constants import voidElements, namespaces
 
 
-class TreeWalker(_base.TreeWalker):
+class TreeWalker(base.TreeWalker):
     def __iter__(self):
         # Buffer the events so we can pass in the following one
         previous = None

html5lib/tests/tokenizer.py

@@ -11,7 +11,7 @@
 from html5lib import html5parser
 from html5lib import treebuilders, serializer, treewalkers
 from html5lib import constants
-from html5lib import utils
+from html5lib import _utils
 
 
 def parse():
@@ -116,7 +116,7 @@ def printOutput(parser, document, opts):
                 import lxml.etree
                 sys.stdout.write(lxml.etree.tostring(document, encoding="unicode"))
             elif tb == "etree":
-                sys.stdout.write(utils.default_etree.tostring(document, encoding="unicode"))
+                sys.stdout.write(_utils.default_etree.tostring(document, encoding="unicode"))
         elif opts.tree:
             if not hasattr(document, '__getitem__'):
                 document = [document]