Skip to content

BUG: Fix insertion of wrong-dtypes NaT into Series[m8ns] #27323

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 10 commits into from
Jul 17, 2019
4 changes: 4 additions & 0 deletions pandas/_libs/index.pyx
Original file line number Diff line number Diff line change
@@ -544,6 +544,10 @@ cpdef convert_scalar(ndarray arr, object value):
pass
elif isinstance(value, timedelta):
return Timedelta(value).value
elif util.is_datetime64_object(value):
# exclude np.datetime64("NaT") which would otherwise be picked up
# by the `value != value check below
pass
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

really everything from 531 down to 555 should be ripped out and this handled by the DatetimeArray/TimedeltaArray __setitem__ implementation

elif value is None or value != value:
return NPY_NAT
elif isinstance(value, str):
7 changes: 5 additions & 2 deletions pandas/core/series.py
Original file line number Diff line number Diff line change
@@ -46,6 +46,7 @@
ABCSparseSeries,
)
from pandas.core.dtypes.missing import (
is_valid_nat_for_dtype,
isna,
na_value_for_dtype,
notna,
@@ -1198,13 +1199,15 @@ def setitem(key, value):
pass
elif is_timedelta64_dtype(self.dtype):
# reassign a null value to iNaT
if isna(value):
if is_valid_nat_for_dtype(value, self.dtype):
# exclude np.datetime64("NaT")
value = iNaT

try:
self.index._engine.set_value(self._values, key, value)
return
except TypeError:
except (TypeError, ValueError):
# ValueError appears in only some builds in CI
pass

self.loc[key] = value
30 changes: 30 additions & 0 deletions pandas/tests/series/indexing/test_indexing.py
Original file line number Diff line number Diff line change
@@ -654,6 +654,36 @@ def test_timedelta_assignment():
tm.assert_series_equal(s, expected)


@pytest.mark.parametrize(
"nat_val,should_cast",
[
(pd.NaT, True),
(np.timedelta64("NaT", "ns"), True),
(np.datetime64("NaT", "ns"), False),
],
)
def test_td64_series_assign_nat(nat_val, should_cast):
# some nat-like values should be cast to timedelta64 when inserting
# into a timedelta64 series. Others should coerce to object
# and retain their dtypes.
base = pd.Series([0, 1, 2], dtype="m8[ns]")
expected = pd.Series([pd.NaT, 1, 2], dtype="m8[ns]")
if not should_cast:
expected = expected.astype(object)

ser = base.copy(deep=True)
ser[0] = nat_val
tm.assert_series_equal(ser, expected)

ser = base.copy(deep=True)
ser.loc[0] = nat_val
tm.assert_series_equal(ser, expected)

ser = base.copy(deep=True)
ser.iloc[0] = nat_val
tm.assert_series_equal(ser, expected)


@pytest.mark.parametrize(
"td",
[