Merge pull request #134 from Nurdok/future-imports

Parse __future__ imports and suppress D302 when `unicode_literals` is imported

Author: Nurdok

docs/release_notes.rst

@@ -12,6 +12,11 @@ New Features
   previously resulted in D100 errors ("Missing docstring in public module")
   will now result in D104 (#105, #127).
 
+Bug Fixes
+
+* On Python 2.x, D302 ("Use u""" for Unicode docstrings") is not reported
+  if `unicode_literals` is imported from `__future__` (#113, #134).
+
 
 0.6.0 - July 20th, 2015
 ---------------------------

requirements/tests.txt

@@ -0,0 +1,3 @@
+pytest==2.7.2
+pytest-pep8
+mock

setup.py

@@ -1,8 +1,9 @@
 from __future__ import with_statement
+import os
 from setuptools import setup
 
 
-with open('pep257.py') as f:
+with open(os.path.join('src', 'pep257.py')) as f:
     for line in f:
         if line.startswith('__version__'):
             version = eval(line.split('=')[-1])
@@ -25,6 +26,7 @@
         'License :: OSI Approved :: MIT License',
     ],
     keywords='PEP 257, pep257, PEP 8, pep8, docstrings',
+    package_dir={'': 'src'},
     py_modules=['pep257'],
-    scripts=['pep257'],
+    scripts=['src/pep257'],
 )

src/__init__.py

@@ -21,6 +21,7 @@
 from optparse import OptionParser
 from re import compile as re
 import itertools
+from collections import defaultdict
 
 try:  # Python 3.x
     from ConfigParser import RawConfigParser
@@ -100,9 +101,9 @@ def __eq__(self, other):
         return other and vars(self) == vars(other)
 
     def __repr__(self):
-        kwargs = ', '.join('{}={!r}'.format(field, getattr(self, field))
+        kwargs = ', '.join('{0}={1!r}'.format(field, getattr(self, field))
                            for field in self._fields)
-        return '{}({})'.format(self.__class__.__name__, kwargs)
+        return '{0}({1})'.format(self.__class__.__name__, kwargs)
 
 
 class Definition(Value):
@@ -131,7 +132,7 @@ def __str__(self):
 class Module(Definition):
 
     _fields = ('name', '_source', 'start', 'end', 'decorators', 'docstring',
-               'children', 'parent', '_all')
+               'children', 'parent', '_all', 'future_imports')
     is_public = True
     _nest = staticmethod(lambda s: {'def': Function, 'class': Class}[s])
     module = property(lambda self: self)
@@ -197,7 +198,7 @@ class Decorator(Value):
 
 class TokenKind(int):
     def __repr__(self):
-        return "tk.{}".format(tk.tok_name[self])
+        return "tk.{0}".format(tk.tok_name[self])
 
 
 class Token(Value):
@@ -251,6 +252,8 @@ def __call__(self, filelike, filename):
         self.stream = TokenStream(StringIO(src))
         self.filename = filename
         self.all = None
+        # TODO: what about Python 3.x?
+        self.future_imports = defaultdict(lambda: False)
         self._accumulated_decorators = []
         return self.parse_module()
 
@@ -349,6 +352,8 @@ def parse_definitions(self, class_, all=False):
             elif self.current.kind == tk.DEDENT:
                 self.consume(tk.DEDENT)
                 return
+            elif self.current.value == 'from':
+                self.parse_from_import_statement()
             else:
                 self.stream.move()
 
@@ -410,6 +415,7 @@ def parse_module(self):
                      [], docstring, children, None, self.all)
         for child in module.children:
             child.parent = module
+        module.future_imports = self.future_imports
         log.debug("finished parsing module.")
         return module
 
@@ -460,6 +466,44 @@ def parse_definition(self, class_):
                   self.current.value)
         return definition
 
+    def parse_from_import_statement(self):
+        """Parse a 'from x import y' statement.
+
+        The purpose is to find __future__ statements.
+
+        """
+        log.debug('parsing from/import statement.')
+        assert self.current.value == 'from', self.current.value
+        self.stream.move()
+        if self.current.value != '__future__':
+            return
+        self.stream.move()
+        assert self.current.value == 'import', self.current.value
+        self.stream.move()
+        if self.current.value == '(':
+            self.consume(tk.OP)
+            expected_end_kind = tk.OP
+        else:
+            expected_end_kind = tk.NEWLINE
+        while self.current.kind != expected_end_kind:
+            if self.current.kind != tk.NAME:
+                self.stream.move()
+                continue
+            log.debug("parsing import, token is %r (%s)",
+                      self.current.kind, self.current.value)
+            log.debug('found future import: %s', self.current.value)
+            self.future_imports[self.current.value] = True
+            self.consume(tk.NAME)
+            log.debug("parsing import, token is %r (%s)",
+                      self.current.kind, self.current.value)
+            if self.current.kind == tk.NAME:
+                self.consume(tk.NAME)  # as
+                self.consume(tk.NAME)  # new name, irrelevant
+            if self.current.value == ',':
+                self.consume(tk.OP)
+            log.debug("parsing import, token is %r (%s)",
+                      self.current.kind, self.current.value)
+
 
 class Error(object):
 
@@ -1123,6 +1167,9 @@ def check_unicode_docstring(self, definition, docstring):
         For Unicode docstrings, use u"""Unicode triple-quoted strings""".
 
         '''
+        if definition.module.future_imports['unicode_literals']:
+            return
+
         # Just check that docstring is unicode, check_triple_double_quotes
         # ensures the correct quotes.
         if docstring and sys.version_info[0] <= 2:

pep257 renamed to src/pep257

@@ -0,0 +1,15 @@
+class Expectation(object):
+    """Hold expectation for pep257 violations in tests."""
+
+    def __init__(self):
+        self.expected = set([])
+
+    def expect(self, *args):
+        """Decorator that expects a certain PEP 257 violation."""
+        def none(_):
+            return None
+
+        if len(args) == 1:
+            return lambda f: (self.expected.add((f.__name__, args[0])) or
+                              none(f()) or f)
+        self.expected.add(args)

pep257.py renamed to src/pep257.py

@@ -1,20 +1,11 @@
 # encoding: utf-8
 # No docstring, so we can test D100
 import sys
+from .expected import Expectation
 
 
-expected = set([])
-
-
-def expect(*args):
-    """Decorator that expects a certain PEP 257 violation."""
-    def none(a):
-        return None
-
-    if len(args) == 1:
-        return lambda f: expected.add((f.__name__, args[0])) or none(f()) or f
-    expected.add(args)
-
+expectation = Expectation()
+expect = expectation.expect
 
 expect('class_', 'D101: Missing docstring in public class')
 
@@ -286,4 +277,12 @@ def a_following_valid_function(x):
 
     """
 
-expect('test.py', 'D100: Missing docstring in public module')
+
+def outer_function():
+    """Do something."""
+    def inner_function():
+        """Do inner something."""
+        return 0
+
+expect(__file__ if __file__[-1] != 'c' else __file__[:-1],
+       'D100: Missing docstring in public module')

src/tests/__init__.py

@@ -0,0 +1,13 @@
+# -*- coding: utf-8 -*-
+"""This is a module."""
+
+from __future__ import unicode_literals
+from .expected import Expectation
+
+
+expectation = Expectation()
+expect = expectation.expect
+
+
+def with_unicode_docstring_without_u():
+    r"""Check unicode: \u2611."""

src/tests/test_cases/__init__.py

@@ -10,7 +10,7 @@
 
 import textwrap
 
-import pep257
+from .. import pep257
 
 
 class TestParser:

src/tests/test_cases/expected.py

@@ -1,5 +1,6 @@
-from pep257 import (StringIO, TokenStream, Parser, Error, check,
-                    Module, Class, Method, Function, NestedFunction)
+import os
+from ..pep257 import (StringIO, TokenStream, Parser, Error, check,
+                      Module, Class, Method, Function, NestedFunction)
 
 
 _ = type('', (), dict(__repr__=lambda *a: '_', __eq__=lambda *a: True))()
@@ -35,14 +36,22 @@ def nested_3(self):
     # Inconvenient comment.
     'a', 'b' 'c',]
 '''
+source_unicode_literals = '''
+from __future__ import unicode_literals
+'''
+source_multiple_future_imports = '''
+from __future__ import (nested_scopes as ns,
+                        unicode_literals)
+'''
 
 
 def test_parser():
     dunder_all = ('a', 'bc')
     module = parse(StringIO(source), 'file.py')
     assert len(list(module)) == 8
     assert Module('file.py', _, 1, len(source.split('\n')),
-                  _, '"""Module."""', _, _, dunder_all) == module
+                  _, '"""Module."""', _, _, dunder_all, {}) == \
+        module
 
     function, class_ = module.children
     assert Function('function', _, _, _, _, '"Function."', _,
@@ -70,12 +79,24 @@ def test_parser():
 
     module = parse(StringIO(source_alt), 'file_alt.py')
     assert Module('file_alt.py', _, 1, len(source_alt.split('\n')),
-                  _, None, _, _, dunder_all) == module
+                  _, None, _, _, dunder_all, {}) == module
 
     module = parse(StringIO(source_alt_nl_at_bracket), 'file_alt_nl.py')
     assert Module('file_alt_nl.py', _, 1,
                   len(source_alt_nl_at_bracket.split('\n')), _, None, _, _,
-                  dunder_all) == module
+                  dunder_all, {}) == module
+
+    module = parse(StringIO(source_unicode_literals), 'file_ucl.py')
+    assert Module('file_ucl.py', _, 1,
+                  _, _, None, _, _,
+                  _, {'unicode_literals': True}) == module
+
+    module = parse(StringIO(source_multiple_future_imports), 'file_mfi.py')
+    assert Module('file_mfi.py', _, 1,
+                  _, _, None, _, _,
+                  _, {'unicode_literals': True, 'nested_scopes': True}) \
+        == module
+    assert module.future_imports['unicode_literals']
 
 
 def _test_module():
@@ -106,11 +127,17 @@ def test_token_stream():
 
 def test_pep257():
     """Run domain-specific tests from test.py file."""
-    import test
-    results = list(check(['test.py']))
-    for error in results:
-        assert isinstance(error, Error)
-    results = set([(e.definition.name, e.message) for e in results])
-    print('\nextra: %r' % (results - test.expected))
-    print('\nmissing: %r' % (test.expected - results))
-    assert test.expected == results
+    test_cases = ('test', 'unicode_literals')
+    for test_case in test_cases:
+        case_module = __import__('test_cases.{0}'.format(test_case),
+                                 globals=globals(),
+                                 locals=locals(),
+                                 fromlist=['expectation'],
+                                 level=1)
+        # from .test_cases import test
+        results = list(check([os.path.join(os.path.dirname(__file__),
+                                           'test_cases', test_case + '.py')]))
+        for error in results:
+            assert isinstance(error, Error)
+        results = set([(e.definition.name, e.message) for e in results])
+        assert case_module.expectation.expected == results

test.py renamed to src/tests/test_cases/test.py

@@ -4,18 +4,18 @@
 
 from __future__ import with_statement
 from collections import namedtuple
+from functools import partial
 
 import sys
 import os
 import mock
 import shlex
-import pytest
 import shutil
 import tempfile
 import textwrap
 import subprocess
 
-import pep257
+from .. import pep257
 
 __all__ = ()
 
@@ -53,11 +53,13 @@ def open(self, path, *args, **kwargs):
 
     def invoke_pep257(self, args=""):
         """Run pep257.py on the environment base folder with the given args."""
-        pep257_location = os.path.join(os.path.dirname(__file__), 'pep257')
+        pep257_location = os.path.join(os.path.dirname(__file__),
+                                       '..', 'pep257')
         cmd = shlex.split("python {0} {1} {2}"
                           .format(pep257_location, self.tempdir, args),
                           posix=False)
-        p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
+        p = subprocess.Popen(cmd,
+                             stdout=subprocess.PIPE,
                              stderr=subprocess.PIPE)
         out, err = p.communicate()
         return self.Result(out=out.decode('utf-8'),
@@ -72,11 +74,13 @@ def __enter__(self):
 
     def __exit__(self, *args, **kwargs):
         shutil.rmtree(self.tempdir)
+        pass
 
 
 def test_pep257_conformance():
-    errors = list(pep257.check(['pep257.py', 'test_pep257.py']))
-    print(errors)
+    relative = partial(os.path.join, os.path.dirname(__file__))
+    errors = list(pep257.check([relative('..', 'pep257.py'),
+                                relative('test_pep257.py')]))
     assert errors == []
 
 
@@ -90,14 +94,15 @@ def function_with_bad_docstring(foo):
     expected_error_codes = set(('D100', 'D400', 'D401', 'D205', 'D209',
                                 'D210'))
     mock_open = mock.mock_open(read_data=function_to_check)
-    with mock.patch('pep257.tokenize_open', mock_open, create=True):
+    from .. import pep257
+    with mock.patch.object(pep257, 'tokenize_open', mock_open, create=True):
         errors = tuple(pep257.check(['filepath']))
         error_codes = set(error.code for error in errors)
         assert error_codes == expected_error_codes
 
     # We need to recreate the mock, otherwise the read file is empty
     mock_open = mock.mock_open(read_data=function_to_check)
-    with mock.patch('pep257.tokenize_open', mock_open, create=True):
+    with mock.patch.object(pep257, 'tokenize_open', mock_open, create=True):
         errors = tuple(pep257.check(['filepath'], ignore=['D100', 'D202']))
         error_codes = set(error.code for error in errors)
         assert error_codes == expected_error_codes - set(('D100', 'D202'))