ENH: fill_value argument for shift #15486

pandas/core/arrays/base.py

@@ -444,8 +444,8 @@ def dropna(self):
         """
         return self[~self.isna()]
 
-    def shift(self, periods=1):
-        # type: (int) -> ExtensionArray
+    def shift(self, periods=1, fill_value=None):
+        # type: (int, object) -> ExtensionArray
         """
         Shift values by desired number.
 
@@ -460,6 +460,12 @@ def shift(self, periods=1):
             The number of periods to shift. Negative values are allowed
             for shifting backwards.
 
+        fill_value : object, optional
+            The scalar value to use for newly introduced missing values.
+            The default is ``self.dtype.na_value``
+
+            .. versionadded:: 0.24.0
+
         Returns
         -------
         shifted : ExtensionArray
@@ -478,8 +484,11 @@ def shift(self, periods=1):
         if not len(self) or periods == 0:
             return self.copy()
 
+        if fill_value is None:
+            fill_value = self.dtype.na_value
+
         empty = self._from_sequence(
-            [self.dtype.na_value] * min(abs(periods), len(self)),
+            [fill_value] * min(abs(periods), len(self)),
             dtype=self.dtype
         )
         if periods > 0:

pandas/core/arrays/categorical.py

@@ -1255,14 +1255,18 @@ def shape(self):
 
         return tuple([len(self._codes)])
 
-    def shift(self, periods):
+    def shift(self, periods, fill_value=None):
         """
         Shift Categorical by desired number of periods.
 
         Parameters
         ----------
         periods : int
             Number of periods to move, can be positive or negative
+        fill_value : object, optional
+            The scalar value to use for newly introduced missing values.
+
+            .. versionadded:: 0.24.0
 
         Returns
         -------
@@ -1275,10 +1279,18 @@ def shift(self, periods):
             raise NotImplementedError("Categorical with ndim > 1.")
         if np.prod(codes.shape) and (periods != 0):
             codes = np.roll(codes, ensure_platform_int(periods), axis=0)
+            if isna(fill_value):
+                fill_value = -1
+            elif fill_value in self.categories:
+                fill_value = self.categories.get_loc(fill_value)
+            else:
+                raise ValueError("'fill_value={}' is not present "
+                                 "in this Categorical's "
+                                 "categories".format(fill_value))
             if periods > 0:
-                codes[:periods] = -1
+                codes[:periods] = fill_value
             else:
-                codes[periods:] = -1
+                codes[periods:] = fill_value
 
         return self.from_codes(codes, categories=self.categories,
                                ordered=self.ordered)

pandas/core/arrays/period.py

@@ -457,7 +457,7 @@ def value_counts(self, dropna=False):
 
     # --------------------------------------------------------------------
 
-    def shift(self, periods=1):
+    def shift(self, periods=1, fill_value=None):
         """
         Shift values by desired number.
 
@@ -471,6 +471,8 @@ def shift(self, periods=1):
         periods : int, default 1
             The number of periods to shift. Negative values are allowed
             for shifting backwards.
+        fill_value : optional, default NaT
+            .. versionadded:: 0.24.0
 
         Returns
         -------
@@ -479,7 +481,7 @@ def shift(self, periods=1):
         # TODO(DatetimeArray): remove
         # The semantics for Index.shift differ from EA.shift
         # then just call super.
-        return ExtensionArray.shift(self, periods)
+        return ExtensionArray.shift(self, periods, fill_value=fill_value)
 
     def _time_shift(self, n, freq=None):
         """

pandas/core/arrays/sparse.py

@@ -887,12 +887,18 @@ def fillna(self, value=None, method=None, limit=None):
 
         return self._simple_new(new_values, self._sparse_index, new_dtype)
 
-    def shift(self, periods=1):
+    def shift(self, periods=1, fill_value=None):
 
         if not len(self) or periods == 0:
             return self.copy()
 
-        subtype = np.result_type(np.nan, self.dtype.subtype)
+        if fill_value is None:
+            res_type = np.nan
+            fill_value = self.dtype.na_value
+        else:
+            res_type = fill_value
+
+        subtype = np.result_type(res_type, self.dtype.subtype)
 
         if subtype != self.dtype.subtype:
             # just coerce up front
@@ -901,7 +907,7 @@ def shift(self, periods=1):
             arr = self
 
         empty = self._from_sequence(
-            [self.dtype.na_value] * min(abs(periods), len(self)),
+            [fill_value] * min(abs(periods), len(self)),
             dtype=arr.dtype
         )
 

pandas/core/frame.py

@@ -3867,9 +3867,9 @@ def replace(self, to_replace=None, value=None, inplace=False, limit=None,
                                               method=method)
 
     @Appender(_shared_docs['shift'] % _shared_doc_kwargs)
-    def shift(self, periods=1, freq=None, axis=0):
+    def shift(self, periods=1, freq=None, axis=0, fill_value=None):
         return super(DataFrame, self).shift(periods=periods, freq=freq,
-                                            axis=axis)
+                                            axis=axis, fill_value=fill_value)
 
     def set_index(self, keys, drop=True, append=False, inplace=False,
                   verify_integrity=False):

pandas/core/generic.py

@@ -8839,6 +8839,12 @@ def mask(self, cond, other=np.nan, inplace=False, axis=None, level=None,
             extend the index when shifting and preserve the original data.
         axis : {0 or 'index', 1 or 'columns', None}, default None
             Shift direction.
+        fill_value : object, optional
+            the scalar value to use for newly introduced missing values.
+            the default depends on the dtype of `self`. for numeric data,
+            ``np.nan`` is used. for datelike, ``pandas.nat`` is used.
+
+            .. versionchanged:: 0.24.0
 
         Returns
         -------
@@ -8874,16 +8880,25 @@ def mask(self, cond, other=np.nan, inplace=False, axis=None, level=None,
         2   NaN  15.0  18.0
         3   NaN  30.0  33.0
         4   NaN  45.0  48.0
+
+        >>> df.shift(periods=3, fill_value=0.0)
+           Col1  Col2  Col3
+        0   0.0   0.0   0.0
+        1   0.0   0.0   0.0
+        2   0.0   0.0   0.0
+        3  10.0  13.0  17.0
+        4  20.0  23.0  27.0
     """)
 
     @Appender(_shared_docs['shift'] % _shared_doc_kwargs)
-    def shift(self, periods=1, freq=None, axis=0):
+    def shift(self, periods=1, freq=None, axis=0, fill_value=None):
         if periods == 0:
             return self.copy()
 
         block_axis = self._get_block_manager_axis(axis)
         if freq is None:
-            new_data = self._data.shift(periods=periods, axis=block_axis)
+            new_data = self._data.shift(periods=periods, axis=block_axis,
+                                        fill_value=fill_value)
         else:
             return self.tshift(periods, freq)
 

pandas/core/groupby/groupby.py

@@ -1992,7 +1992,7 @@ def _get_cythonized_result(self, how, grouper, aggregate=False,
 
     @Substitution(name='groupby')
     @Appender(_doc_template)
-    def shift(self, periods=1, freq=None, axis=0):
+    def shift(self, periods=1, freq=None, axis=0, fill_value=None):
         """
         Shift each group by periods observations.
 
@@ -2002,10 +2002,13 @@ def shift(self, periods=1, freq=None, axis=0):
             number of periods to shift
         freq : frequency string
         axis : axis to shift, default 0
+        fill_value : optional
+            .. versionadded:: 0.24.0
         """
 
-        if freq is not None or axis != 0:
-            return self.apply(lambda x: x.shift(periods, freq, axis))
+        if freq is not None or axis != 0 or fill_value is not None:
+            return self.apply(lambda x: x.shift(periods, freq,
+                                                axis, fill_value))
 
         return self._get_cythonized_result('group_shift_indexer',
                                            self.grouper, cython_dtype=np.int64,

pandas/core/internals/blocks.py

@@ -1260,12 +1260,12 @@ def diff(self, n, axis=1):
         new_values = algos.diff(self.values, n, axis=axis)
         return [self.make_block(values=new_values)]
 
-    def shift(self, periods, axis=0):
+    def shift(self, periods, axis=0, fill_value=None):
         """ shift the block by periods, possibly upcast """
 
         # convert integer to float if necessary. need to do a lot more than
         # that, handle boolean etc also
-        new_values, fill_value = maybe_upcast(self.values)
+        new_values, fill_value = maybe_upcast(self.values, fill_value)
 
         # make sure array sent to np.roll is c_contiguous
         f_ordered = new_values.flags.f_contiguous
@@ -1955,17 +1955,19 @@ def interpolate(self, method='pad', axis=0, inplace=False, limit=None,
                                  limit=limit),
             placement=self.mgr_locs)
 
-    def shift(self, periods, axis=0):
+    def shift(self, periods, axis=0, fill_value=None):
         """
         Shift the block by `periods`.
 
         Dispatches to underlying ExtensionArray and re-boxes in an
         ExtensionBlock.
         """
         # type: (int, Optional[BlockPlacement]) -> List[ExtensionBlock]
-        return [self.make_block_same_class(self.values.shift(periods=periods),
-                                           placement=self.mgr_locs,
-                                           ndim=self.ndim)]
+        return [
+            self.make_block_same_class(
+                self.values.shift(periods=periods, fill_value=fill_value),
+                placement=self.mgr_locs, ndim=self.ndim)
+        ]
 
     @property
     def _ftype(self):
@@ -2945,7 +2947,7 @@ def _try_coerce_result(self, result):
     def _box_func(self):
         return lambda x: tslibs.Timestamp(x, tz=self.dtype.tz)
 
-    def shift(self, periods, axis=0):
+    def shift(self, periods, axis=0, fill_value=None):
         """ shift the block by periods """
 
         # think about moving this to the DatetimeIndex. This is a non-freq
@@ -2960,10 +2962,12 @@ def shift(self, periods, axis=0):
 
         new_values = self.values.asi8.take(indexer)
 
+        if fill_value is None:
+            fill_value = tslibs.iNaT
         if periods > 0:
-            new_values[:periods] = tslibs.iNaT
+            new_values[:periods] = fill_value
         else:
-            new_values[periods:] = tslibs.iNaT
+            new_values[periods:] = fill_value
 
         new_values = self.values._shallow_copy(new_values)
         return [self.make_block_same_class(new_values,

pandas/core/series.py

@@ -3714,8 +3714,9 @@ def replace(self, to_replace=None, value=None, inplace=False, limit=None,
                                            regex=regex, method=method)
 
     @Appender(generic._shared_docs['shift'] % _shared_doc_kwargs)
-    def shift(self, periods=1, freq=None, axis=0):
-        return super(Series, self).shift(periods=periods, freq=freq, axis=axis)
+    def shift(self, periods=1, freq=None, axis=0, fill_value=None):
+        return super(Series, self).shift(periods=periods, freq=freq, axis=axis,
+                                         fill_value=fill_value)
 
     def reindex_axis(self, labels, axis=0, **kwargs):
         """

pandas/tests/arrays/sparse/test_array.py

@@ -262,6 +262,20 @@ def test_take_negative(self):
         exp = SparseArray(np.take(self.arr_data, [-4, -3, -2]))
         tm.assert_sp_array_equal(self.arr.take([-4, -3, -2]), exp)
 
+    def test_shift_fill_value(self):
+        # GH #24128
+        fill_values = [0, None, np.nan]
+        for fill_value in fill_values:
+            print(fill_value)
+            sparse = SparseArray(np.array([1, 0, 0, 3, 0]),
+                                 fill_value=8.0)
+            res = sparse.shift(1, fill_value=fill_value)
+            if fill_value is None:
+                fill_value = res.dtype.na_value
+            exp = SparseArray(np.array([fill_value, 1, 0, 0, 3]),
+                              fill_value=8.0)
+            tm.assert_sp_array_equal(res, exp)
+
     def test_bad_take(self):
         with pytest.raises(IndexError, match="bounds"):
             self.arr.take([11])

pandas/tests/frame/test_timeseries.py

@@ -320,6 +320,20 @@ def test_shift_categorical(self):
         xp = DataFrame({'one': s1.shift(1), 'two': s2.shift(1)})
         assert_frame_equal(rs, xp)
 
+    def test_shift_fill_value(self):
+        # GH #24128
+        df = DataFrame([1, 2, 3, 4, 5],
+                       index=date_range('1/1/2000', periods=5, freq='H'))
+        exp = DataFrame([0, 1, 2, 3, 4],
+                        index=date_range('1/1/2000', periods=5, freq='H'))
+        result = df.shift(1, fill_value=0)
+        assert_frame_equal(result, exp)
+
+        exp = DataFrame([0, 0, 1, 2, 3],
+                        index=date_range('1/1/2000', periods=5, freq='H'))
+        result = df.shift(2, fill_value=0)
+        assert_frame_equal(result, exp)
+
     def test_shift_empty(self):
         # Regression test for #8019
         df = DataFrame({'foo': []})

pandas/tests/groupby/test_groupby.py

@@ -1617,6 +1617,23 @@ def test_group_shift_with_null_key():
     assert_frame_equal(result, expected)
 
 
+def test_group_shift_with_fill_value():
+    # GH #24128
+    n_rows = 24
+    df = DataFrame([(i % 12, i % 3, i)
+                    for i in range(n_rows)], dtype=float,
+                   columns=["A", "B", "Z"], index=None)
+    g = df.groupby(["A", "B"])
+
+    expected = DataFrame([(i + 12 if i < n_rows - 12
+                           else 0)
+                          for i in range(n_rows)], dtype=float,
+                         columns=["Z"], index=None)
+    result = g.shift(-1, fill_value=0)[["Z"]]
+
+    assert_frame_equal(result, expected)
+
+
 def test_pivot_table_values_key_error():
     # This test is designed to replicate the error in issue #14938
     df = pd.DataFrame({'eventDate':

pandas/tests/series/test_timeseries.py

@@ -129,6 +129,34 @@ def test_shift2(self):
         idx = DatetimeIndex(['2000-01-01', '2000-01-02', '2000-01-04'])
         pytest.raises(NullFrequencyError, idx.shift, 1)
 
+    def test_shift_fill_value(self):
+        # GH #24128
+        ts = Series([1.0, 2.0, 3.0, 4.0, 5.0],
+                    index=date_range('1/1/2000', periods=5, freq='H'))
+
+        exp = Series([0.0, 1.0, 2.0, 3.0, 4.0],
+                     index=date_range('1/1/2000', periods=5, freq='H'))
+        # check that fill value works
+        result = ts.shift(1, fill_value=0.0)
+        tm.assert_series_equal(result, exp)
+
+        exp = Series([0.0, 0.0, 1.0, 2.0, 3.0],
+                     index=date_range('1/1/2000', periods=5, freq='H'))
+        result = ts.shift(2, fill_value=0.0)
+        tm.assert_series_equal(result, exp)
+
+    def test_categorical_shift_fill_value(self):
+        ts = pd.Series(['a', 'b', 'c', 'd'], dtype="category")
+        res = ts.shift(1, fill_value='a')
+        expected = pd.Series(pd.Categorical(['a', 'a', 'b', 'c'],
+                                            categories=['a', 'b', 'c', 'd'],
+                                            ordered=False))
+        tm.assert_equal(res, expected)
+
+        # check for incorrect fill_value
+        with pytest.raises(ValueError):
+            ts.shift(1, fill_value='f')
+
     def test_shift_dst(self):
         # GH 13926
         dates = date_range('2016-11-06', freq='H', periods=10, tz='US/Eastern')