Skip to content

DEPR: fillna method kwd #53496

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 14 commits into from
Jun 7, 2023
1 change: 1 addition & 0 deletions ci/code_checks.sh
Original file line number Diff line number Diff line change
@@ -289,6 +289,7 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
pandas.core.groupby.SeriesGroupBy.cumprod \
pandas.core.groupby.SeriesGroupBy.cumsum \
pandas.core.groupby.SeriesGroupBy.diff \
pandas.core.groupby.SeriesGroupBy.fillna \
pandas.core.groupby.SeriesGroupBy.ffill \
pandas.core.groupby.SeriesGroupBy.max \
pandas.core.groupby.SeriesGroupBy.median \
2 changes: 1 addition & 1 deletion doc/source/getting_started/comparison/includes/missing.rst
Original file line number Diff line number Diff line change
@@ -19,7 +19,7 @@ Forward fill from previous rows

.. ipython:: python

outer_join.fillna(method="ffill")
outer_join.ffill()

Replace missing values with a specified value
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
4 changes: 2 additions & 2 deletions doc/source/user_guide/basics.rst
Original file line number Diff line number Diff line change
@@ -1377,12 +1377,12 @@ These methods require that the indexes are **ordered** increasing or
decreasing.

Note that the same result could have been achieved using
:ref:`fillna <missing_data.fillna>` (except for ``method='nearest'``) or
:ref:`ffill <missing_data.fillna>` (except for ``method='nearest'``) or
:ref:`interpolate <missing_data.interpolate>`:

.. ipython:: python

ts2.reindex(ts.index).fillna(method="ffill")
ts2.reindex(ts.index).ffill()

:meth:`~Series.reindex` will raise a ValueError if the index is not monotonically
increasing or decreasing. :meth:`~Series.fillna` and :meth:`~Series.interpolate`
6 changes: 3 additions & 3 deletions doc/source/user_guide/missing_data.rst
Original file line number Diff line number Diff line change
@@ -145,7 +145,7 @@ objects.
:suppress:

df = df2.loc[:, ["one", "two", "three"]]
a = df2.loc[df2.index[:5], ["one", "two"]].fillna(method="pad")
a = df2.loc[df2.index[:5], ["one", "two"]].ffill()
b = df2.loc[df2.index[:5], ["one", "two", "three"]]

.. ipython:: python
@@ -237,7 +237,7 @@ can propagate non-NA values forward or backward:
.. ipython:: python

df
df.fillna(method="pad")
df.ffill()

.. _missing_data.fillna.limit:

@@ -254,7 +254,7 @@ we can use the ``limit`` keyword:
.. ipython:: python

df
df.fillna(method="pad", limit=1)
df.ffill(limit=1)

To remind you, these are the available filling methods:

1 change: 1 addition & 0 deletions doc/source/whatsnew/v0.10.0.rst
Original file line number Diff line number Diff line change
@@ -226,6 +226,7 @@ labeled the aggregated group with the end of the interval: the next day).
valid code. You must either specify a fill value or an interpolation method:

.. ipython:: python
:okwarning:

s = pd.Series([np.nan, 1.0, 2.0, np.nan, 4])
s
2 changes: 2 additions & 0 deletions doc/source/whatsnew/v2.1.0.rst
Original file line number Diff line number Diff line change
@@ -279,6 +279,8 @@ Deprecations
- Deprecated constructing :class:`SparseArray` from scalar data, pass a sequence instead (:issue:`53039`)
- Deprecated option "mode.use_inf_as_na", convert inf entries to ``NaN`` before instead (:issue:`51684`)
- Deprecated positional indexing on :class:`Series` with :meth:`Series.__getitem__` and :meth:`Series.__setitem__`, in a future version ``ser[item]`` will *always* interpret ``item`` as a label, not a position (:issue:`50617`)
- Deprecated the "method" and "limit" keywords on :meth:`Series.fillna`, :meth:`DataFrame.fillna`, :meth:`SeriesGroupBy.fillna`, :meth:`DataFrameGroupBy.fillna`, and :meth:`Resampler.fillna`, use ``obj.bfill()`` or ``obj.ffill()`` instead (:issue:`53394`)
-

.. ---------------------------------------------------------------------------
.. _whatsnew_210.performance:
12 changes: 12 additions & 0 deletions pandas/conftest.py
Original file line number Diff line number Diff line change
@@ -150,6 +150,18 @@ def pytest_collection_modifyitems(items, config) -> None:
"first is deprecated and will be removed in a future version. "
"Please create a mask and filter using `.loc` instead",
),
(
"Resampler.fillna",
"DatetimeIndexResampler.fillna is deprecated",
),
(
"DataFrameGroupBy.fillna",
"DataFrameGroupBy.fillna with 'method' is deprecated",
),
(
"DataFrameGroupBy.fillna",
"DataFrame.fillna with 'method' is deprecated",
),
]

for item in items:
154 changes: 109 additions & 45 deletions pandas/core/generic.py
Original file line number Diff line number Diff line change
@@ -6803,6 +6803,61 @@ def convert_dtypes(
# ----------------------------------------------------------------------
# Filling NA's

def _deprecate_downcast(self, downcast) -> None:
if isinstance(downcast, dict):
# GH#40988
for dc in downcast.values():
if dc is not None and dc is not False and dc != "infer":
warnings.warn(
"downcast entries other than None, False, and 'infer' "
"are deprecated and will raise in a future version",
FutureWarning,
stacklevel=find_stack_level(),
)
elif downcast is not None and downcast is not False and downcast != "infer":
# GH#40988
warnings.warn(
"downcast other than None, False, and 'infer' are deprecated "
"and will raise in a future version",
FutureWarning,
stacklevel=find_stack_level(),
)

@final
def _fillna_with_method(
self,
method: Literal["ffill", "bfill", "pad", "backfill"],
*,
axis: None | Axis = None,
inplace: bool_t = False,
limit: None | int = None,
downcast: dict | None = None,
):
if axis is None:
axis = 0
axis = self._get_axis_number(axis)
method = clean_fill_method(method)

if not self._mgr.is_single_block and axis == 1:
if inplace:
raise NotImplementedError()
result = self.T._fillna_with_method(method=method, limit=limit).T

return result

new_mgr = self._mgr.interpolate(
method=method,
axis=axis,
limit=limit,
inplace=inplace,
downcast=downcast,
)
result = self._constructor(new_mgr)
if inplace:
return self._update_inplace(result)
else:
return result.__finalize__(self, method="fillna")

@overload
def fillna(
self,
@@ -6874,6 +6929,9 @@ def fillna(
* ffill: propagate last valid observation forward to next valid.
* backfill / bfill: use next valid observation to fill gap.

.. deprecated:: 2.1.0
Use ffill or bfill instead.

axis : {axes_single_arg}
Axis along which to fill missing values. For `Series`
this parameter is unused and defaults to 0.
@@ -6927,15 +6985,6 @@ def fillna(
2 0.0 0.0 0.0 0.0
3 0.0 3.0 0.0 4.0

We can also propagate non-null values forward or backward.

>>> df.fillna(method="ffill")
A B C D
0 NaN 2.0 NaN 0.0
1 3.0 4.0 NaN 1.0
2 3.0 4.0 NaN 1.0
3 3.0 3.0 NaN 4.0

Replace all NaN elements in column 'A', 'B', 'C', and 'D', with 0, 1,
2, and 3 respectively.

@@ -6971,42 +7020,29 @@ def fillna(
"""
inplace = validate_bool_kwarg(inplace, "inplace")
value, method = validate_fillna_kwargs(value, method)

if isinstance(downcast, dict):
# GH#40988
for dc in downcast.values():
if dc is not None and dc is not False and dc != "infer":
warnings.warn(
"downcast entries other than None, False, and 'infer' "
"are deprecated and will raise in a future version",
FutureWarning,
stacklevel=find_stack_level(),
)
elif downcast is not None and downcast is not False and downcast != "infer":
# GH#40988
if method is not None:
warnings.warn(
"downcast other than None, False, and 'infer' are deprecated "
"and will raise in a future version",
f"{type(self).__name__}.fillna with 'method' is deprecated and "
"will raise in a future version. Use obj.ffill() or obj.bfill() "
"instead.",
FutureWarning,
stacklevel=find_stack_level(),
)

self._deprecate_downcast(downcast)

# set the default here, so functions examining the signaure
# can detect if something was set (e.g. in groupby) (GH9221)
if axis is None:
axis = 0
axis = self._get_axis_number(axis)

if value is None:
if not self._mgr.is_single_block and axis == 1:
if inplace:
raise NotImplementedError()
result = self.T.fillna(method=method, limit=limit).T

return result

new_data = self._mgr.interpolate(
method=method,
return self._fillna_with_method(
# error: Argument 1 to "_fillna_with_method" of "NDFrame" has
# incompatible type "Optional[Literal['backfill', 'bfill', 'ffill',
# 'pad']]"; expected "Literal['ffill', 'bfill', 'pad', 'backfill']"
method, # type: ignore[arg-type]
axis=axis,
limit=limit,
inplace=inplace,
@@ -7111,7 +7147,10 @@ def fillna(
if axis == 1:
result = self.T.fillna(value=value, limit=limit).T

new_data = result
# error: Incompatible types in assignment (expression
# has type "Self", variable has type "Union[ArrayManager,
# SingleArrayManager, BlockManager, SingleBlockManager]")
new_data = result # type: ignore[assignment]
else:
new_data = self._mgr.fillna(
value=value, limit=limit, inplace=inplace, downcast=downcast
@@ -7180,6 +7219,25 @@ def ffill(

Examples
--------
>>> df = pd.DataFrame([[np.nan, 2, np.nan, 0],
... [3, 4, np.nan, 1],
... [np.nan, np.nan, np.nan, np.nan],
... [np.nan, 3, np.nan, 4]],
... columns=list("ABCD"))
>>> df
A B C D
0 NaN 2.0 NaN 0.0
1 3.0 4.0 NaN 1.0
2 NaN NaN NaN NaN
3 NaN 3.0 NaN 4.0

>>> df.ffill()
A B C D
0 NaN 2.0 NaN 0.0
1 3.0 4.0 NaN 1.0
2 3.0 4.0 NaN 1.0
3 3.0 3.0 NaN 4.0

>>> ser = pd.Series([1, np.NaN, 2, 3])
>>> ser.ffill()
0 1.0
@@ -7188,8 +7246,10 @@ def ffill(
3 3.0
dtype: float64
"""
return self.fillna(
method="ffill", axis=axis, inplace=inplace, limit=limit, downcast=downcast
self._deprecate_downcast(downcast)

return self._fillna_with_method(
"ffill", axis=axis, inplace=inplace, limit=limit, downcast=downcast
)

@final
@@ -7319,8 +7379,9 @@ def bfill(
2 4.0 7
3 4.0 7
"""
return self.fillna(
method="bfill", axis=axis, inplace=inplace, limit=limit, downcast=downcast
self._deprecate_downcast(downcast)
return self._fillna_with_method(
"bfill", axis=axis, inplace=inplace, limit=limit, downcast=downcast
)

@final
@@ -9840,8 +9901,8 @@ def _align_frame(
)

if method is not None:
left = left.fillna(method=method, axis=fill_axis, limit=limit)
right = right.fillna(method=method, axis=fill_axis, limit=limit)
left = left._fillna_with_method(method, axis=fill_axis, limit=limit)
right = right._fillna_with_method(method, axis=fill_axis, limit=limit)

return left, right, join_index

@@ -9916,8 +9977,13 @@ def _align_series(
# fill
fill_na = notna(fill_value) or (method is not None)
if fill_na:
left = left.fillna(fill_value, method=method, limit=limit, axis=fill_axis)
right = right.fillna(fill_value, method=method, limit=limit)
fill_value, method = validate_fillna_kwargs(fill_value, method)
if method is not None:
left = left._fillna_with_method(method, limit=limit, axis=fill_axis)
right = right._fillna_with_method(method, limit=limit)
else:
left = left.fillna(fill_value, limit=limit, axis=fill_axis)
right = right.fillna(fill_value, limit=limit)

return left, right, join_index

@@ -11283,9 +11349,7 @@ def pct_change(
if fill_method is None:
data = self
else:
_data = self.fillna(method=fill_method, axis=axis, limit=limit)
assert _data is not None # needed for mypy
data = _data
data = self._fillna_with_method(fill_method, axis=axis, limit=limit)

shifted = data.shift(periods=periods, freq=freq, axis=axis, **kwargs)
# Unsupported left operand type for / ("Self")
Loading