ENH: fill_value argument for shift #15486

pandas/core/frame.py

@@ -3867,9 +3867,9 @@ def replace(self, to_replace=None, value=None, inplace=False, limit=None,
                                               method=method)
 
     @Appender(_shared_docs['shift'] % _shared_doc_kwargs)
-    def shift(self, periods=1, freq=None, axis=0):
+    def shift(self, periods=1, freq=None, axis=0, fill_value=None):
         return super(DataFrame, self).shift(periods=periods, freq=freq,
-                                            axis=axis)
+                                            axis=axis, fill_value=fill_value)
 
     def set_index(self, keys, drop=True, append=False, inplace=False,
                   verify_integrity=False):

pandas/core/generic.py

@@ -8830,6 +8830,12 @@ def mask(self, cond, other=np.nan, inplace=False, axis=None, level=None,
             extend the index when shifting and preserve the original data.
         axis : {0 or 'index', 1 or 'columns', None}, default None
             Shift direction.
+
+        .. versionchanged:: 0.24.0
+
+            fill_value : None or value, default None (NaN).
+                Filling the empty space after shift with value provided by 
+                the user instead of NaN
 
         Returns
         -------
@@ -8865,16 +8871,27 @@ def mask(self, cond, other=np.nan, inplace=False, axis=None, level=None,
         2   NaN  15.0  18.0
         3   NaN  30.0  33.0
         4   NaN  45.0  48.0
+
+        .. versionchanged:: 0.24.0
+
+            >>> df.shift(periods=3, fill_value=0.0)
+               Col1  Col2  Col3
+            0   0.0   0.0   0.0
+            1   0.0   0.0   0.0
+            2   0.0   0.0   0.0
+            3  10.0  13.0  17.0
+            4  20.0  23.0  27.0
     """)
 
     @Appender(_shared_docs['shift'] % _shared_doc_kwargs)
-    def shift(self, periods=1, freq=None, axis=0):
+    def shift(self, periods=1, freq=None, axis=0, fill_value=None):
         if periods == 0:
             return self.copy()
 
         block_axis = self._get_block_manager_axis(axis)
         if freq is None:
-            new_data = self._data.shift(periods=periods, axis=block_axis)
+            new_data = self._data.shift(periods=periods, axis=block_axis,
+                                        fill_value=fill_value)
         else:
             return self.tshift(periods, freq)
 

pandas/core/groupby/groupby.py

@@ -1993,7 +1993,7 @@ def _get_cythonized_result(self, how, grouper, aggregate=False,
 
     @Substitution(name='groupby')
     @Appender(_doc_template)
-    def shift(self, periods=1, freq=None, axis=0):
+    def shift(self, periods=1, freq=None, axis=0, fill_value=None):
         """
         Shift each group by periods observations.
 
@@ -2003,10 +2003,11 @@ def shift(self, periods=1, freq=None, axis=0):
             number of periods to shift
         freq : frequency string
         axis : axis to shift, default 0
+        fill_value : None or value, default None (NaN).
         """
 
         if freq is not None or axis != 0:
-            return self.apply(lambda x: x.shift(periods, freq, axis))
+            return self.apply(lambda x: x.shift(periods, freq, axis, fill_value))
 
         return self._get_cythonized_result('group_shift_indexer',
                                            self.grouper, cython_dtype=np.int64,

pandas/core/internals/blocks.py

@@ -1260,12 +1260,12 @@ def diff(self, n, axis=1):
         new_values = algos.diff(self.values, n, axis=axis)
         return [self.make_block(values=new_values)]
 
-    def shift(self, periods, axis=0):
+    def shift(self, periods, axis=0, fill_value=None):
         """ shift the block by periods, possibly upcast """
 
         # convert integer to float if necessary. need to do a lot more than
         # that, handle boolean etc also
-        new_values, fill_value = maybe_upcast(self.values)
+        new_values, fill_value = maybe_upcast(self.values, fill_value)
 
         # make sure array sent to np.roll is c_contiguous
         f_ordered = new_values.flags.f_contiguous

pandas/core/series.py

@@ -3714,8 +3714,9 @@ def replace(self, to_replace=None, value=None, inplace=False, limit=None,
                                            regex=regex, method=method)
 
     @Appender(generic._shared_docs['shift'] % _shared_doc_kwargs)
-    def shift(self, periods=1, freq=None, axis=0):
-        return super(Series, self).shift(periods=periods, freq=freq, axis=axis)
+    def shift(self, periods=1, freq=None, axis=0, fill_value=None):
+        return super(Series, self).shift(periods=periods, freq=freq, axis=axis,
+                                         fill_value=fill_value)
 
     def reindex_axis(self, labels, axis=0, **kwargs):
         """

pandas/tests/frame/test_timeseries.py

@@ -320,6 +320,20 @@ def test_shift_categorical(self):
         xp = DataFrame({'one': s1.shift(1), 'two': s2.shift(1)})
         assert_frame_equal(rs, xp)
 
+    def test_shift_fill_value(self):
+        # GH #24128
+        df = DataFrame(np.random.randnint(5), index=date_range('1/1/2000',
+                                                            periods=5,
+                                                            freq='H'))
+        result = df.shift(1, fill_value=0)
+        tm.assert_equal(result.iloc[0, 0].value, 0)
+        tm.assert_equal(result.iloc[1, 0].value, df.iloc[0, 0].value)
+
+        result = df.shift(2, fill_value=0)
+        tm.assert_equal(result.iloc[0, 0].value, 0)
+        tm.assert_equal(result.iloc[1, 0].value, 0)
+        tm.assert_equal(result.iloc[2, 0].value, df.iloc[0, 0].value)
+
     def test_shift_empty(self):
         # Regression test for #8019
         df = DataFrame({'foo': []})

pandas/tests/series/test_timeseries.py

@@ -129,6 +129,27 @@ def test_shift2(self):
         idx = DatetimeIndex(['2000-01-01', '2000-01-02', '2000-01-04'])
         pytest.raises(NullFrequencyError, idx.shift, 1)
 
+    def test_shift_fill_value(self):
+        # GH #24128
+        ts = Series(np.random.randn(5),
+                    index=date_range('1/1/2000', periods=5, freq='H'))
+
+        # fill_value should have no effect on shift with freq
+        result = ts.shift(1, freq='5T', fill_value=0)
+        exp_index = ts.index.shift(1, freq='5T')
+        tm.assert_index_equal(result.index, exp_index)
+        tm.assert_equal(result.iloc[0].value, ts.iloc[0].value)
+
+        # check that fill value works
+        result = ts.shift(1, fill_value=0.0)
+        tm.assert_equal(result.iloc[0].value, 0.0)
+        tm.assert_equal(result.iloc[1].value, ts.iloc[0].value)
+
+        result = ts.shift(2, fill_value=0.0)
+        tm.assert_equal(result.iloc[0].value, 0.0)
+        tm.assert_equal(result.iloc[1].value, 0.0)
+        tm.assert_equal(result.iloc[2].value, ts.iloc[0].value)
+
     def test_shift_dst(self):
         # GH 13926
         dates = date_range('2016-11-06', freq='H', periods=10, tz='US/Eastern')