Merge pull request #83 from davidastephens/url_encode_cleanup

ENH: Add _encode_url to improve readability of url construction

Author: davidastephens

pandas_datareader/google/daily.py

@@ -1,9 +1,9 @@
 from pandas.io.common import urlencode
-from pandas_datareader.utils import _retry_read_url
+from pandas_datareader.utils import _retry_read_url, _encode_url
 from pandas_datareader.utils import _sanitize_dates
 from pandas_datareader.utils import _get_data_from
 
-_URL = 'http://www.google.com/finance/historical?'
+_URL = 'http://www.google.com/finance/historical'
 
 
 def _get_data(symbols=None, start=None, end=None, retry_count=3,
@@ -49,9 +49,11 @@ def _get_data_one(sym, start, end, interval, retry_count, pause):
     start, end = _sanitize_dates(start, end)
 
     # www.google.com/finance/historical?q=GOOG&startdate=Jun+9%2C+2011&enddate=Jun+8%2C+2013&output=csv
-    url = "%s%s" % (_URL,
-                    urlencode({"q": sym,
-                               "startdate": start.strftime('%b %d, ' '%Y'),
-                               "enddate": end.strftime('%b %d, %Y'),
-                               "output": "csv"}))
+    params = {
+        'q': sym,
+        'startdate': start.strftime('%b %d, %Y'),
+        'enddate': end.strftime('%b %d, %Y'),
+        'output': "csv"
+    }
+    url = _encode_url(_URL, params)
     return _retry_read_url(url, retry_count, pause, 'Google')

pandas_datareader/utils.py

@@ -12,6 +12,10 @@
 from pandas.compat import StringIO, bytes_to_str
 from pandas.util.testing import _network_error_classes
 
+if compat.PY3:
+    from urllib.parse import urlencode
+else:
+    from urllib import urlencode
 
 class SymbolWarning(UserWarning):
     pass
@@ -85,6 +89,16 @@ def _in_chunks(seq, size):
     """
     return (seq[pos:pos + size] for pos in range(0, len(seq), size))
 
+def _encode_url(url, params):
+    """
+    Return encoded url with parameters
+    """
+    s_params = urlencode(params)
+    if s_params:
+        return url + '?' + s_params
+    else:
+        return url
+
 def _retry_read_url(url, retry_count, pause, name):
     """
     Open url (and retry)

pandas_datareader/yahoo/actions.py

@@ -5,9 +5,9 @@
 from pandas.util.testing import _network_error_classes
 from pandas.compat import StringIO, bytes_to_str
 
-from pandas_datareader.utils import _sanitize_dates
+from pandas_datareader.utils import _sanitize_dates, _encode_url
 
-_URL = 'http://ichart.finance.yahoo.com/x?'
+_URL = 'http://ichart.finance.yahoo.com/x'
 
 
 def _get_data(symbol, start=None, end=None, retry_count=3, pause=0.001):
@@ -31,14 +31,17 @@ def _get_data(symbol, start=None, end=None, retry_count=3, pause=0.001):
     """
 
     start, end = _sanitize_dates(start, end)
-    url = (_URL + 's=%s' % symbol + \
-                '&a=%s' % (start.month - 1) + \
-                '&b=%s' % start.day + \
-                '&c=%s' % start.year + \
-                '&d=%s' % (end.month - 1) + \
-                '&e=%s' % end.day + \
-                '&f=%s' % end.year + \
-                '&g=v')
+    params = {
+        's': symbol,
+        'a': start.month - 1,
+        'b': start.day,
+        'c': start.year,
+        'd': end.month - 1,
+        'e': end.day,
+        'f': end.year,
+        'g': 'v'
+    }
+    url = _encode_url(_URL, params)
 
     for _ in range(retry_count):
         time.sleep(pause)

pandas_datareader/yahoo/daily.py

@@ -1,8 +1,8 @@
-from pandas_datareader.utils import _retry_read_url
+from pandas_datareader.utils import _retry_read_url, _encode_url
 from pandas_datareader.utils import _sanitize_dates
 from pandas_datareader.utils import _get_data_from
 
-_URL = 'http://ichart.finance.yahoo.com/table.csv?'
+_URL = 'http://ichart.finance.yahoo.com/table.csv'
 
 def _get_data(symbols=None, start=None, end=None, retry_count=3,
                    pause=0.001, adjust_price=False, ret_index=False,
@@ -96,13 +96,16 @@ def _get_data_one(sym, start, end, interval, retry_count, pause):
     Returns a DataFrame.
     """
     start, end = _sanitize_dates(start, end)
-    url = (_URL + 's=%s' % sym +
-           '&a=%s' % (start.month - 1) +
-           '&b=%s' % start.day +
-           '&c=%s' % start.year +
-           '&d=%s' % (end.month - 1) +
-           '&e=%s' % end.day +
-           '&f=%s' % end.year +
-           '&g=%s' % interval +
-           '&ignore=.csv')
+    params = {
+        's': sym,
+        'a': start.month - 1,
+        'b': start.day,
+        'c': start.year,
+        'd': end.month - 1,
+        'e': end.day,
+        'f': end.year,
+        'g': interval,
+        'ignore': '.csv'
+    }
+    url = _encode_url(_URL, params)
     return _retry_read_url(url, retry_count, pause, 'Yahoo!')

pandas_datareader/yahoo/quotes.py

@@ -2,12 +2,13 @@
 import pandas.compat as compat
 from pandas.io.common import urlopen
 from pandas import DataFrame
+from pandas_datareader.utils import _encode_url
 
 _yahoo_codes = {'symbol': 's', 'last': 'l1', 'change_pct': 'p2', 'PE': 'r',
                 'time': 't1', 'short_ratio': 's7'}
 
 
-_URL = 'http://finance.yahoo.com/d/quotes.csv?'
+_URL = 'http://finance.yahoo.com/d/quotes.csv'
 
 
 def _get_data(symbols):
@@ -27,10 +28,14 @@ def _get_data(symbols):
 
     data = defaultdict(list)
 
-    url_str = _URL + 's=%s&f=%s' % (sym_list, request)
+    params = {
+        's': sym_list,
+        'f': request
+    }
+    url = _encode_url(_URL, params)
 
-    with urlopen(url_str) as url:
-        lines = url.readlines()
+    with urlopen(url) as response:
+        lines = response.readlines()
 
     for line in lines:
         fields = line.decode('utf-8').strip().split(',')