Merge pull request #39 from python-hyper/query_value_encoding

Allow unescaped equal signs in query parameter values

Author: mahmoud

.tox-coveragerc

@@ -3,6 +3,8 @@ branch = True
 source =
     hyperlink
     ../hyperlink
+omit =
+    */flycheck_*
 
 [paths]
 source =

hyperlink/_url.py

@@ -169,8 +169,10 @@ def __nonzero__(self):
 _SCHEMELESS_PATH_DELIMS = _ALL_DELIMS - _SCHEMELESS_PATH_SAFE
 _FRAGMENT_SAFE = _UNRESERVED_CHARS | _PATH_SAFE | set(u'/?')
 _FRAGMENT_DELIMS = _ALL_DELIMS - _FRAGMENT_SAFE
-_QUERY_SAFE = _UNRESERVED_CHARS | _FRAGMENT_SAFE - set(u'&=+')
-_QUERY_DELIMS = _ALL_DELIMS - _QUERY_SAFE
+_QUERY_VALUE_SAFE = _UNRESERVED_CHARS | _FRAGMENT_SAFE - set(u'&+')
+_QUERY_VALUE_DELIMS = _ALL_DELIMS - _QUERY_VALUE_SAFE
+_QUERY_KEY_SAFE = _UNRESERVED_CHARS | _QUERY_VALUE_SAFE - set(u'=')
+_QUERY_KEY_DELIMS = _ALL_DELIMS - _QUERY_KEY_SAFE
 
 
 def _make_decode_map(delims, allow_percent=False):
@@ -204,8 +206,10 @@ def _make_quote_map(safe_chars):
 _PATH_PART_QUOTE_MAP = _make_quote_map(_PATH_SAFE)
 _SCHEMELESS_PATH_PART_QUOTE_MAP = _make_quote_map(_SCHEMELESS_PATH_SAFE)
 _PATH_DECODE_MAP = _make_decode_map(_PATH_DELIMS)
-_QUERY_PART_QUOTE_MAP = _make_quote_map(_QUERY_SAFE)
-_QUERY_DECODE_MAP = _make_decode_map(_QUERY_DELIMS)
+_QUERY_KEY_QUOTE_MAP = _make_quote_map(_QUERY_KEY_SAFE)
+_QUERY_KEY_DECODE_MAP = _make_decode_map(_QUERY_KEY_DELIMS)
+_QUERY_VALUE_QUOTE_MAP = _make_quote_map(_QUERY_VALUE_SAFE)
+_QUERY_VALUE_DECODE_MAP = _make_decode_map(_QUERY_VALUE_DELIMS)
 _FRAGMENT_QUOTE_MAP = _make_quote_map(_FRAGMENT_SAFE)
 _FRAGMENT_DECODE_MAP = _make_decode_map(_FRAGMENT_DELIMS)
 _UNRESERVED_QUOTE_MAP = _make_quote_map(_UNRESERVED_CHARS)
@@ -290,17 +294,28 @@ def _encode_path_parts(text_parts, rooted=False, has_scheme=True,
     return tuple(encoded_parts)
 
 
-def _encode_query_part(text, maximal=True):
+def _encode_query_key(text, maximal=True):
     """
     Percent-encode a single query string key or value.
     """
     if maximal:
         bytestr = normalize('NFC', text).encode('utf8')
-        return u''.join([_QUERY_PART_QUOTE_MAP[b] for b in bytestr])
-    return u''.join([_QUERY_PART_QUOTE_MAP[t] if t in _QUERY_DELIMS else t
+        return u''.join([_QUERY_KEY_QUOTE_MAP[b] for b in bytestr])
+    return u''.join([_QUERY_KEY_QUOTE_MAP[t] if t in _QUERY_KEY_DELIMS else t
                      for t in text])
 
 
+def _encode_query_value(text, maximal=True):
+    """
+    Percent-encode a single query string key or value.
+    """
+    if maximal:
+        bytestr = normalize('NFC', text).encode('utf8')
+        return u''.join([_QUERY_VALUE_QUOTE_MAP[b] for b in bytestr])
+    return u''.join([_QUERY_VALUE_QUOTE_MAP[t]
+                     if t in _QUERY_VALUE_DELIMS else t for t in text])
+
+
 def _encode_fragment_part(text, maximal=True):
     """Quote the fragment part of the URL. Fragments don't have
     subdelimiters, so the whole URL fragment can be passed.
@@ -498,10 +513,16 @@ def _decode_path_part(text, normalize_case=False, encode_stray_percents=False):
                            _decode_map=_PATH_DECODE_MAP)
 
 
-def _decode_query_part(text, normalize_case=False, encode_stray_percents=False):
+def _decode_query_key(text, normalize_case=False, encode_stray_percents=False):
+    return _percent_decode(text, normalize_case=normalize_case,
+                           encode_stray_percents=encode_stray_percents,
+                           _decode_map=_QUERY_KEY_DECODE_MAP)
+
+
+def _decode_query_value(text, normalize_case=False, encode_stray_percents=False):
     return _percent_decode(text, normalize_case=normalize_case,
                            encode_stray_percents=encode_stray_percents,
-                           _decode_map=_QUERY_DECODE_MAP)
+                           _decode_map=_QUERY_VALUE_DECODE_MAP)
 
 
 def _decode_fragment_part(text, normalize_case=False, encode_stray_percents=False):
@@ -1341,9 +1362,9 @@ def to_uri(self):
             userinfo=new_userinfo,
             host=new_host,
             path=new_path,
-            query=tuple([tuple(_encode_query_part(x, maximal=True)
-                               if x is not None else None
-                               for x in (k, v))
+            query=tuple([(_encode_query_key(k, maximal=True),
+                          _encode_query_value(v, maximal=True)
+                          if v is not None else None)
                          for k, v in self.query]),
             fragment=_encode_fragment_part(self.fragment, maximal=True)
         )
@@ -1379,9 +1400,9 @@ def to_iri(self):
                             host=host_text,
                             path=[_decode_path_part(segment)
                                   for segment in self.path],
-                            query=[tuple(_decode_query_part(x)
-                                         if x is not None else None
-                                         for x in (k, v))
+                            query=[(_decode_query_key(k),
+                                    _decode_query_value(v)
+                                    if v is not None else None)
                                    for k, v in self.query],
                             fragment=_decode_fragment_part(self.fragment))
 
@@ -1417,10 +1438,14 @@ def to_text(self, with_password=False):
                                   has_scheme=bool(scheme),
                                   has_authority=bool(authority),
                                   maximal=False)
-        query_string = u'&'.join(
-            u'='.join((_encode_query_part(x, maximal=False)
-                       for x in ([k] if v is None else [k, v])))
-            for (k, v) in self.query)
+        query_parts = []
+        for k, v in self.query:
+            if v is None:
+                query_parts.append(_encode_query_key(k, maximal=False))
+            else:
+                query_parts.append(u'='.join((_encode_query_key(k, maximal=False),
+                                              _encode_query_value(v, maximal=False))))
+        query_string = u'&'.join(query_parts)
 
         fragment = self.fragment
 

hyperlink/test/test_url.py

@@ -542,10 +542,16 @@ def test_parseEqualSignInParamValue(self):
         """
         u = URL.from_text('http://localhost/?=x=x=x')
         self.assertEqual(u.get(''), ['x=x=x'])
-        self.assertEqual(u.to_text(), 'http://localhost/?=x%3Dx%3Dx')
+        self.assertEqual(u.to_text(), 'http://localhost/?=x=x=x')
         u = URL.from_text('http://localhost/?foo=x=x=x&bar=y')
         self.assertEqual(u.query, (('foo', 'x=x=x'), ('bar', 'y')))
-        self.assertEqual(u.to_text(), 'http://localhost/?foo=x%3Dx%3Dx&bar=y')
+        self.assertEqual(u.to_text(), 'http://localhost/?foo=x=x=x&bar=y')
+
+        u = URL.from_text('https://example.com/?argument=3&argument=4&operator=%3D')
+        iri = u.to_iri()
+        self.assertEqual(iri.get('operator'), ['='])
+        # assert that the equals is not unnecessarily escaped
+        self.assertEqual(iri.to_uri().get('operator'), ['='])
 
     def test_empty(self):
         """

tox.ini

@@ -4,7 +4,7 @@ envlist = py26,py27,py34,py35,py36,pypy,coverage-report,packaging
 [testenv]
 changedir = .tox
 deps = -rrequirements-test.txt
-commands = coverage run --parallel --omit 'flycheck__*' --rcfile {toxinidir}/.tox-coveragerc -m pytest --doctest-modules {envsitepackagesdir}/hyperlink {posargs}
+commands = coverage run --parallel --rcfile {toxinidir}/.tox-coveragerc -m pytest --doctest-modules {envsitepackagesdir}/hyperlink {posargs}
 
 # Uses default basepython otherwise reporting doesn't work on Travis where
 # Python 3.6 is only available in 3.6 jobs.