WIP: Index no longer an ndarray sub-class (GH5080)

CLN: add searchsorted to core/base (GH6712, GH7447, GH6469)

fixup tests in test_timeseries for reverse ndarray/datetimeindex comparisons

fix algos / multi-index repeat (essentially this is a bug-fix)

ENH: add NumericIndex and operators, related (GH7439)

Author: jreback

pandas/compat/pickle_compat.py

@@ -7,27 +7,30 @@
 import pickle as pkl
 from pandas import compat
 from pandas.compat import u, string_types
-from pandas.core.series import Series, TimeSeries
-from pandas.sparse.series import SparseSeries, SparseTimeSeries
-
 
 def load_reduce(self):
     stack = self.stack
     args = stack.pop()
     func = stack[-1]
+
     if type(args[0]) is type:
         n = args[0].__name__
-        if n == u('DeprecatedSeries') or n == u('DeprecatedTimeSeries'):
-            stack[-1] = object.__new__(Series)
-            return
-        elif (n == u('DeprecatedSparseSeries') or
-              n == u('DeprecatedSparseTimeSeries')):
-            stack[-1] = object.__new__(SparseSeries)
-            return
 
     try:
-        value = func(*args)
-    except:
+        stack[-1] = func(*args)
+        return
+    except Exception as e:
+
+        # if we have a deprecated function
+        # try to replace and try again
+
+        if '_reconstruct: First argument must be a sub-type of ndarray' in str(e):
+            try:
+                cls = args[0]
+                stack[-1] = object.__new__(cls)
+                return
+            except:
+                pass
 
         # try to reencode the arguments
         if getattr(self,'encoding',None) is not None:
@@ -57,6 +60,21 @@ class Unpickler(pkl.Unpickler):
 Unpickler.dispatch = copy.copy(Unpickler.dispatch)
 Unpickler.dispatch[pkl.REDUCE[0]] = load_reduce
 
+def load_newobj(self):
+    args = self.stack.pop()
+    cls = self.stack[-1]
+
+    try:
+        obj = cls.__new__(cls, *args)
+    except (Exception) as e:
+
+        # replace the default newobj creator to
+        # handle an odd issue with 0-len series
+        if not len(args):
+            obj = object.__new__(cls)
+
+    self.stack[-1] = obj
+Unpickler.dispatch[pkl.NEWOBJ[0]] = load_newobj
 
 def load(fh, encoding=None, compat=False, is_verbose=False):
     """load a pickle, with a provided encoding
@@ -74,11 +92,6 @@ def load(fh, encoding=None, compat=False, is_verbose=False):
     """
 
     try:
-        if compat:
-            pandas.core.series.Series = DeprecatedSeries
-            pandas.core.series.TimeSeries = DeprecatedTimeSeries
-            pandas.sparse.series.SparseSeries = DeprecatedSparseSeries
-            pandas.sparse.series.SparseTimeSeries = DeprecatedSparseTimeSeries
         fh.seek(0)
         if encoding is not None:
             up = Unpickler(fh, encoding=encoding)
@@ -89,25 +102,3 @@ def load(fh, encoding=None, compat=False, is_verbose=False):
         return up.load()
     except:
         raise
-    finally:
-        if compat:
-            pandas.core.series.Series = Series
-            pandas.core.series.Series = TimeSeries
-            pandas.sparse.series.SparseSeries = SparseSeries
-            pandas.sparse.series.SparseTimeSeries = SparseTimeSeries
-
-
-class DeprecatedSeries(np.ndarray, Series):
-    pass
-
-
-class DeprecatedTimeSeries(DeprecatedSeries):
-    pass
-
-
-class DeprecatedSparseSeries(DeprecatedSeries):
-    pass
-
-
-class DeprecatedSparseTimeSeries(DeprecatedSparseSeries):
-    pass

pandas/core/base.py

@@ -8,7 +8,7 @@
 from pandas.core import common as com
 import pandas.core.nanops as nanops
 import pandas.tslib as tslib
-from pandas.util.decorators import cache_readonly
+from pandas.util.decorators import Appender, cache_readonly
 
 class StringMixin(object):
 
@@ -205,6 +205,19 @@ def __unicode__(self):
                                  quote_strings=True)
         return "%s(%s, dtype='%s')" % (type(self).__name__, prepr, self.dtype)
 
+def _unbox(func):
+    @Appender(func.__doc__)
+    def f(self, *args, **kwargs):
+        result = func(self.values, *args, **kwargs)
+        from pandas.core.index import Index
+        if isinstance(result, (np.ndarray, com.ABCSeries, Index)) and result.ndim == 0:
+            # return NumPy type
+            return result.dtype.type(result.item())
+        else:  # pragma: no cover
+            return result
+    f.__name__ = func.__name__
+    return f
+
 class IndexOpsMixin(object):
     """ common ops mixin to support a unified inteface / docs for Series / Index """
 
@@ -340,6 +353,20 @@ def factorize(self, sort=False, na_sentinel=-1):
         from pandas.core.algorithms import factorize
         return factorize(self, sort=sort, na_sentinel=na_sentinel)
 
+    def searchsorted(self, key, side='left'):
+        """ np.ndarray searchsorted compat """
+
+        ### FIXME ###
+        #### needs coercion on the key (DatetimeIndex does alreaY)
+        #### needs tests/doc-string ###
+        return self.values.searchsorted(key, side=side)
+
+    #----------------------------------------------------------------------
+    # unbox reductions
+
+    all = _unbox(np.ndarray.all)
+    any = _unbox(np.ndarray.any)
+
 # facilitate the properties on the wrapped ops
 def _field_accessor(name, docstring=None):
     op_accessor = '_{0}'.format(name)
@@ -456,6 +483,19 @@ def min(self, axis=None):
         except ValueError:
             return self._na_value
 
+    def argmin(self, axis=None):
+        """
+        return the minimum argument indexer
+
+        FIXME: need some tests (what do do if all NaT?)
+        """
+        i8 = self.asi8
+        if self.hasnans:
+            mask = i8 == tslib.iNaT
+            i8 = i8.copy()
+            i8[mask] = np.iinfo('int64').max
+        return i8.argmin()
+
     def max(self, axis=None):
         """
         Overridden ndarray.max to return an object
@@ -477,6 +517,19 @@ def max(self, axis=None):
         except ValueError:
             return self._na_value
 
+    def argmax(self, axis=None):
+        """
+        return the maximum argument indexer
+
+        FIXME: need some tests (what do do if all NaT?)
+        """
+        i8 = self.asi8
+        if self.hasnans:
+            mask = i8 == tslib.iNaT
+            i8 = i8.copy()
+            i8[mask] = 0
+        return i8.argmax()
+
     @property
     def _formatter_func(self):
         """

pandas/core/categorical.py

@@ -939,6 +939,6 @@ def _get_codes_for_values(values, levels):
         levels = com._ensure_object(levels)
     (hash_klass, vec_klass), vals = _get_data_algo(values, _hashtables)
     t = hash_klass(len(levels))
-    t.map_locations(levels)
+    t.map_locations(com._values_from_object(levels))
     return com._ensure_platform_int(t.lookup(values))
 

pandas/core/common.py

@@ -205,7 +205,7 @@ def _isnull_new(obj):
     # hack (for now) because MI registers as ndarray
     elif isinstance(obj, pd.MultiIndex):
         raise NotImplementedError("isnull is not defined for MultiIndex")
-    elif isinstance(obj, (ABCSeries, np.ndarray)):
+    elif isinstance(obj, (ABCSeries, np.ndarray, pd.Index)):
         return _isnull_ndarraylike(obj)
     elif isinstance(obj, ABCGeneric):
         return obj._constructor(obj._data.isnull(func=isnull))
@@ -231,7 +231,7 @@ def _isnull_old(obj):
     # hack (for now) because MI registers as ndarray
     elif isinstance(obj, pd.MultiIndex):
         raise NotImplementedError("isnull is not defined for MultiIndex")
-    elif isinstance(obj, (ABCSeries, np.ndarray)):
+    elif isinstance(obj, (ABCSeries, np.ndarray, pd.Index)):
         return _isnull_ndarraylike_old(obj)
     elif isinstance(obj, ABCGeneric):
         return obj._constructor(obj._data.isnull(func=_isnull_old))
@@ -2024,8 +2024,7 @@ def _is_bool_indexer(key):
 def _default_index(n):
     from pandas.core.index import Int64Index
     values = np.arange(n, dtype=np.int64)
-    result = values.view(Int64Index)
-    result.name = None
+    result = Int64Index(values,name=None)
     result.is_unique = True
     return result
 

pandas/core/format.py

@@ -1186,7 +1186,7 @@ def _helper_csv(self, writer, na_rep=None, cols=None,
         if cols is None:
             cols = self.columns
 
-        has_aliases = isinstance(header, (tuple, list, np.ndarray))
+        has_aliases = isinstance(header, (tuple, list, np.ndarray, Index))
         if has_aliases or header:
             if index:
                 # should write something for index label
@@ -1205,7 +1205,7 @@ def _helper_csv(self, writer, na_rep=None, cols=None,
                             else:
                                 index_label = [index_label]
                     elif not isinstance(index_label,
-                                        (list, tuple, np.ndarray)):
+                                        (list, tuple, np.ndarray, Index)):
                         # given a string for a DF with Index
                         index_label = [index_label]
 
@@ -1327,7 +1327,7 @@ def _save_header(self):
         header = self.header
         encoded_labels = []
 
-        has_aliases = isinstance(header, (tuple, list, np.ndarray))
+        has_aliases = isinstance(header, (tuple, list, np.ndarray, Index))
         if not (has_aliases or self.header):
             return
         if has_aliases:
@@ -1355,7 +1355,7 @@ def _save_header(self):
                             index_label = ['']
                         else:
                             index_label = [index_label]
-                elif not isinstance(index_label, (list, tuple, np.ndarray)):
+                elif not isinstance(index_label, (list, tuple, np.ndarray, Index)):
                     # given a string for a DF with Index
                     index_label = [index_label]
 
@@ -1520,7 +1520,7 @@ def _format_value(self, val):
         return val
 
     def _format_header_mi(self):
-        has_aliases = isinstance(self.header, (tuple, list, np.ndarray))
+        has_aliases = isinstance(self.header, (tuple, list, np.ndarray, Index))
         if not(has_aliases or self.header):
             return
 
@@ -1566,7 +1566,7 @@ def _format_header_mi(self):
         self.rowcounter = lnum
 
     def _format_header_regular(self):
-        has_aliases = isinstance(self.header, (tuple, list, np.ndarray))
+        has_aliases = isinstance(self.header, (tuple, list, np.ndarray, Index))
         if has_aliases or self.header:
             coloffset = 0
 
@@ -1611,7 +1611,7 @@ def _format_body(self):
             return self._format_regular_rows()
 
     def _format_regular_rows(self):
-        has_aliases = isinstance(self.header, (tuple, list, np.ndarray))
+        has_aliases = isinstance(self.header, (tuple, list, np.ndarray, Index))
         if has_aliases or self.header:
             self.rowcounter += 1
 
@@ -1621,7 +1621,7 @@ def _format_regular_rows(self):
             # chek aliases
             # if list only take first as this is not a MultiIndex
             if self.index_label and isinstance(self.index_label,
-                                               (list, tuple, np.ndarray)):
+                                               (list, tuple, np.ndarray, Index)):
                 index_label = self.index_label[0]
             # if string good to go
             elif self.index_label and isinstance(self.index_label, str):
@@ -1661,7 +1661,7 @@ def _format_regular_rows(self):
                 yield ExcelCell(self.rowcounter + i, colidx + coloffset, val)
 
     def _format_hierarchical_rows(self):
-        has_aliases = isinstance(self.header, (tuple, list, np.ndarray))
+        has_aliases = isinstance(self.header, (tuple, list, np.ndarray, Index))
         if has_aliases or self.header:
             self.rowcounter += 1
 
@@ -1671,7 +1671,7 @@ def _format_hierarchical_rows(self):
             index_labels = self.df.index.names
             # check for aliases
             if self.index_label and isinstance(self.index_label,
-                                               (list, tuple, np.ndarray)):
+                                               (list, tuple, np.ndarray, Index)):
                 index_labels = self.index_label
 
             # if index labels are not empty go ahead and dump