Merge remote-tracking branch 'pandas-dev/master'

Author: jaehoonhwang

.travis.yml

@@ -86,6 +86,7 @@ matrix:
         - JOB_NAME: "36"
         - TEST_ARGS="--skip-slow --skip-network"
         - PANDAS_TESTING_MODE="deprecate"
+        - CONDA_FORGE=true
       addons:
         apt:
           packages:
@@ -122,11 +123,6 @@ matrix:
         - PANDAS_TESTING_MODE="deprecate"
         - CACHE_NAME="35_numpy_dev"
         - USE_CACHE=true
-      addons:
-        apt:
-          packages:
-          - libatlas-base-dev
-          - gfortran
 #    In allow_failures
     - python: 3.5
       env:
@@ -166,11 +162,6 @@ matrix:
           - PANDAS_TESTING_MODE="deprecate"
           - CACHE_NAME="35_numpy_dev"
           - USE_CACHE=true
-        addons:
-          apt:
-            packages:
-            - libatlas-base-dev
-            - gfortran
       - python: 3.5
         env:
         - PYTHON_VERSION=3.5

README.md

@@ -41,8 +41,8 @@
 <tr>
   <td></td>
   <td>
-    <a href="https://ci.appveyor.com/project/jreback/pandas-465">
-    <img src="https://ci.appveyor.com/api/projects/status/iblk29s98quexwxi/branch/master?svg=true" alt="appveyor build status" />
+    <a href="https://ci.appveyor.com/project/pandas-dev/pandas">
+    <img src="https://ci.appveyor.com/api/projects/status/86vn83mxgnl4xf1s/branch/master?svg=true" alt="appveyor build status" />
     </a>
   </td>
 </tr>

asv_bench/benchmarks/join_merge.py

@@ -6,7 +6,7 @@
     from pandas import ordered_merge as merge_ordered
 
 
-#----------------------------------------------------------------------
+# ----------------------------------------------------------------------
 # Append
 
 class Append(object):
@@ -35,7 +35,7 @@ def time_append_mixed(self):
         self.mdf1.append(self.mdf2)
 
 
-#----------------------------------------------------------------------
+# ----------------------------------------------------------------------
 # Concat
 
 class Concat(object):
@@ -120,7 +120,7 @@ def time_f_ordered_axis1(self):
         concat(self.frames_f, axis=1, ignore_index=True)
 
 
-#----------------------------------------------------------------------
+# ----------------------------------------------------------------------
 # Joins
 
 class Join(object):
@@ -202,7 +202,7 @@ def time_join_non_unique_equal(self):
         (self.fracofday * self.temp[self.fracofday.index])
 
 
-#----------------------------------------------------------------------
+# ----------------------------------------------------------------------
 # Merges
 
 class Merge(object):
@@ -257,7 +257,31 @@ def time_i8merge(self):
         merge(self.left, self.right, how='outer')
 
 
-#----------------------------------------------------------------------
+class MergeCategoricals(object):
+    goal_time = 0.2
+
+    def setup(self):
+        self.left_object = pd.DataFrame(
+            {'X': np.random.choice(range(0, 10), size=(10000,)),
+             'Y': np.random.choice(['one', 'two', 'three'], size=(10000,))})
+
+        self.right_object = pd.DataFrame(
+            {'X': np.random.choice(range(0, 10), size=(10000,)),
+             'Z': np.random.choice(['jjj', 'kkk', 'sss'], size=(10000,))})
+
+        self.left_cat = self.left_object.assign(
+            Y=self.left_object['Y'].astype('category'))
+        self.right_cat = self.right_object.assign(
+            Z=self.right_object['Z'].astype('category'))
+
+    def time_merge_object(self):
+        merge(self.left_object, self.right_object, on='X')
+
+    def time_merge_cat(self):
+        merge(self.left_cat, self.right_cat, on='X')
+
+
+# ----------------------------------------------------------------------
 # Ordered merge
 
 class MergeOrdered(object):
@@ -332,7 +356,7 @@ def time_multiby(self):
         merge_asof(self.df1e, self.df2e, on='time', by=['key', 'key2'])
 
 
-#----------------------------------------------------------------------
+# ----------------------------------------------------------------------
 # data alignment
 
 class Align(object):

ci/build_docs.sh

@@ -23,9 +23,6 @@ if [ x"$DOC_BUILD" != x"" ]; then
 
     source activate pandas
 
-    # install sudo deps
-    time sudo apt-get $APT_ARGS install dvipng texlive-latex-base texlive-latex-extra
-
     mv "$TRAVIS_BUILD_DIR"/doc /tmp
     cd /tmp/doc
 

ci/install_travis.sh

@@ -53,14 +53,17 @@ conda config --set ssl_verify false || exit 1
 conda config --set always_yes true --set changeps1 false || exit 1
 conda update -q conda
 
+echo "[add channels]"
 # add the pandas channel to take priority
 # to add extra packages
-echo "[add channels]"
 conda config --add channels pandas || exit 1
 conda config --remove channels defaults || exit 1
 conda config --add channels defaults || exit 1
 
-conda install anaconda-client
+if [ "$CONDA_FORGE" ]; then
+    # add conda-forge channel as priority
+    conda config --add channels conda-forge || exit 1
+fi
 
 # Useful for debugging any issues with conda
 conda info -a || exit 1

ci/lint.sh

@@ -8,9 +8,9 @@ RET=0
 
 if [ "$LINT" ]; then
 
-    # pandas/src is C code, so no need to search there.
+    # pandas/_libs/src is C code, so no need to search there.
     echo "Linting  *.py"
-    flake8 pandas --filename=*.py --exclude pandas/src
+    flake8 pandas --filename=*.py --exclude pandas/_libs/src
     if [ $? -ne "0" ]; then
         RET=1
     fi
@@ -46,8 +46,8 @@ if [ "$LINT" ]; then
     echo "Linting *.c and *.h"
     for path in '*.h' 'period_helper.c' 'datetime' 'parser' 'ujson'
     do
-        echo "linting -> pandas/src/$path"
-        cpplint --quiet --extensions=c,h --headers=h --filter=-readability/casting,-runtime/int,-build/include_subdir --recursive pandas/src/$path
+        echo "linting -> pandas/_libs/src/$path"
+        cpplint --quiet --extensions=c,h --headers=h --filter=-readability/casting,-runtime/int,-build/include_subdir --recursive pandas/_libs/src/$path
         if [ $? -ne "0" ]; then
             RET=1
         fi

ci/requirements-3.5_NUMPY_DEV.build.sh

@@ -8,6 +8,7 @@ echo "install numpy master wheel"
 pip uninstall numpy -y
 
 # install numpy wheel from master
-pip install --pre --upgrade --no-index --timeout=60 --trusted-host travis-dev-wheels.scipy.org -f http://travis-dev-wheels.scipy.org/ numpy scipy
+PRE_WHEELS="https://7933911d6844c6c53a7d-47bd50c35cd79bd838daf386af554a83.ssl.cf2.rackcdn.com"
+pip install --pre --upgrade --timeout=60 -f $PRE_WHEELS numpy scipy
 
 true

ci/requirements-3.6.run

@@ -14,6 +14,7 @@ html5lib
 jinja2
 sqlalchemy
 pymysql
+feather-format
 # psycopg2 (not avail on defaults ATM)
 beautifulsoup4
 s3fs

ci/requirements-3.6.sh

@@ -4,5 +4,4 @@ numpy
 cython
 pytest
 pytest-cov
-pytest-xdist
 flake8

ci/requirements_dev.txt

@@ -197,7 +197,7 @@ def html():
                 print(e)
                 print("Failed to convert %s" % nb)
 
-    if os.system('sphinx-build -j 2 -P -b html -d build/doctrees '
+    if os.system('sphinx-build -P -b html -d build/doctrees '
                  'source build/html'):
         raise SystemExit("Building HTML failed.")
     try:

doc/make.py

@@ -118,7 +118,6 @@ Google BigQuery
    :toctree: generated/
 
    read_gbq
-   to_gbq
 
 
 .. currentmodule:: pandas
@@ -712,8 +711,8 @@ Serialization / IO / Conversion
    Series.to_string
    Series.to_clipboard
 
-Sparse methods
-~~~~~~~~~~~~~~
+Sparse
+~~~~~~
 .. autosummary::
    :toctree: generated/
 
@@ -1031,6 +1030,13 @@ Serialization / IO / Conversion
    DataFrame.to_string
    DataFrame.to_clipboard
 
+Sparse
+~~~~~~
+.. autosummary::
+   :toctree: generated/
+
+   SparseDataFrame.to_coo
+
 .. _api.panel:
 
 Panel
@@ -1237,7 +1243,7 @@ Serialization / IO / Conversion
    Panel.to_frame
    Panel.to_xarray
    Panel.to_clipboard
-   
+
 .. _api.index:
 
 Index
@@ -1405,6 +1411,7 @@ MultiIndex
    :toctree: generated/
 
    MultiIndex
+   IndexSlice
 
 MultiIndex Components
 ~~~~~~~~~~~~~~~~~~~~~~

doc/source/api.rst

@@ -617,6 +617,7 @@ Assigning a `Categorical` to parts of a column of other types will use the value
     df
     df.dtypes
 
+.. _categorical.merge:
 
 Merging
 ~~~~~~~
@@ -646,6 +647,9 @@ In this case the categories are not the same and so an error is raised:
 
 The same applies to ``df.append(df_different)``.
 
+See also the section on :ref:`merge dtypes<merging.dtypes>` for notes about preserving merge dtypes and performance.
+
+
 .. _categorical.union:
 
 Unioning

doc/source/categorical.rst

@@ -16,6 +16,14 @@
 import inspect
 from pandas.compat import u, PY3
 
+# https://github.com/sphinx-doc/sphinx/pull/2325/files
+# Workaround for sphinx-build recursion limit overflow:
+# pickle.dump(doctree, f, pickle.HIGHEST_PROTOCOL)
+#  RuntimeError: maximum recursion depth exceeded while pickling an object
+#
+# Python's default allowed recursion depth is 1000.
+sys.setrecursionlimit(5000)
+
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
@@ -46,7 +54,7 @@
               'ipython_sphinxext.ipython_console_highlighting',
               'sphinx.ext.intersphinx',
               'sphinx.ext.coverage',
-              'sphinx.ext.pngmath',
+              'sphinx.ext.mathjax',
               'sphinx.ext.ifconfig',
               'sphinx.ext.linkcode',
               ]

doc/source/conf.py

@@ -116,7 +116,6 @@ See the package overview for more detail about what's in the library.
     whatsnew
     install
     contributing
-    faq
     overview
     10min
     tutorials

doc/source/index.rst.template

@@ -260,7 +260,7 @@ Optional Dependencies
   <http://www.vergenet.net/~conrad/software/xsel/>`__, or `xclip
   <https://github.com/astrand/xclip/>`__: necessary to use
   :func:`~pandas.read_clipboard`. Most package managers on Linux distributions will have ``xclip`` and/or ``xsel`` immediately available for installation.
-* For Google BigQuery I/O - see :ref:`here <io.bigquery_deps>`.
+* For Google BigQuery I/O - see `here <https://pandas-gbq.readthedocs.io/en/latest/install.html#dependencies>`__
 
 * `Backports.lzma <https://pypi.python.org/pypi/backports.lzma/>`__: Only for Python 2, for writing to and/or reading from an xz compressed DataFrame in CSV; Python 3 support is built into the standard library.
 * One of the following combinations of libraries is needed to use the