From 3fa3e814ecd8bda72eb8a358069093b007dd30ef Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Thu, 14 Feb 2019 18:52:48 -0800 Subject: [PATCH 1/4] Fix+test #25282, #25317 --- doc/source/whatsnew/v0.24.2.rst | 3 ++- pandas/core/arrays/datetimes.py | 6 +++--- pandas/core/arrays/timedeltas.py | 5 +++++ pandas/tests/arithmetic/test_datetime64.py | 14 +++++++++++++ pandas/tests/arrays/test_timedeltas.py | 23 ++++++++++++++++++++++ 5 files changed, 47 insertions(+), 4 deletions(-) diff --git a/doc/source/whatsnew/v0.24.2.rst b/doc/source/whatsnew/v0.24.2.rst index f17c4974cd450..cf86c008f29e8 100644 --- a/doc/source/whatsnew/v0.24.2.rst +++ b/doc/source/whatsnew/v0.24.2.rst @@ -25,6 +25,7 @@ Fixed Regressions - Fixed regression in :meth:`DataFrame.apply` causing ``RecursionError`` when ``dict``-like classes were passed as argument. (:issue:`25196`) - Fixed regression in :meth:`DataFrame.duplicated()`, where empty dataframe was not returning a boolean dtyped Series. (:issue:`25184`) +- Fixed regression in subtraction between :class:`Series` objects with ``datetime64[ns]`` dtype incorrectly raising ``OverflowError`` when the `Series` on the right contains null values (:issue:`25317`) .. _whatsnew_0242.enhancements: @@ -71,7 +72,7 @@ Bug Fixes - **Timedelta** - +- Bug in :class:`TimedeltaIndex` where `np.sum(index)` incorrectly returned a zero-dimensional object instead of a scalar (:issue:`25282`) - - - diff --git a/pandas/core/arrays/datetimes.py b/pandas/core/arrays/datetimes.py index 1b2a4da389dc4..8ad15abd9a5b9 100644 --- a/pandas/core/arrays/datetimes.py +++ b/pandas/core/arrays/datetimes.py @@ -720,11 +720,11 @@ def _sub_datetime_arraylike(self, other): self_i8 = self.asi8 other_i8 = other.asi8 + arr_mask = self._isnan | other._isnan new_values = checked_add_with_arr(self_i8, -other_i8, - arr_mask=self._isnan) + arr_mask=arr_mask) if self._hasnans or other._hasnans: - mask = (self._isnan) | (other._isnan) - new_values[mask] = iNaT + new_values[arr_mask] = iNaT return new_values.view('timedelta64[ns]') def _add_offset(self, offset): diff --git a/pandas/core/arrays/timedeltas.py b/pandas/core/arrays/timedeltas.py index 06e2bf76fcf96..74fe8072e6924 100644 --- a/pandas/core/arrays/timedeltas.py +++ b/pandas/core/arrays/timedeltas.py @@ -190,6 +190,8 @@ def __init__(self, values, dtype=_TD_DTYPE, freq=None, copy=False): "ndarray, or Series or Index containing one of those." ) raise ValueError(msg.format(type(values).__name__)) + if values.ndim != 1: + raise ValueError("Only 1-dimensional input arrays are supported.") if values.dtype == 'i8': # for compat with datetime/timedelta/period shared methods, @@ -945,6 +947,9 @@ def sequence_to_td64ns(data, copy=False, unit="ns", errors="raise"): .format(dtype=data.dtype)) data = np.array(data, copy=copy) + if data.ndim != 1: + raise ValueError("Only 1-dimensional input arrays are supported.") + assert data.dtype == 'm8[ns]', data return data, inferred_freq diff --git a/pandas/tests/arithmetic/test_datetime64.py b/pandas/tests/arithmetic/test_datetime64.py index 405dc0805a285..c81a371f37dc1 100644 --- a/pandas/tests/arithmetic/test_datetime64.py +++ b/pandas/tests/arithmetic/test_datetime64.py @@ -1440,6 +1440,20 @@ def test_dt64arr_add_sub_offset_ndarray(self, tz_naive_fixture, class TestDatetime64OverflowHandling(object): # TODO: box + de-duplicate + def test_dt64_overflow_masking(self, box_with_array): + # GH#25317 + left = Series([Timestamp('1969-12-31')]) + right = Series([NaT]) + + left = tm.box_expected(left, box_with_array) + right = tm.box_expected(right, box_with_array) + + expected = TimedeltaIndex([NaT]) + expected = tm.box_expected(expected, box_with_array) + + result = left - right + tm.assert_equal(result, expected) + def test_dt64_series_arith_overflow(self): # GH#12534, fixed by GH#19024 dt = pd.Timestamp('1700-01-31') diff --git a/pandas/tests/arrays/test_timedeltas.py b/pandas/tests/arrays/test_timedeltas.py index 6b4662ca02e80..85004b106633c 100644 --- a/pandas/tests/arrays/test_timedeltas.py +++ b/pandas/tests/arrays/test_timedeltas.py @@ -9,6 +9,18 @@ class TestTimedeltaArrayConstructor(object): + def test_only_1dim_accepted(self): + # GH#25282 + arr = np.array([0, 1, 2, 3], dtype='m8[h]').astype('m8[ns]') + + with pytest.raises(ValueError, match="Only 1-dimensional"): + # 2-dim + TimedeltaArray(arr.reshape(2, 2)) + + with pytest.raises(ValueError, match="Only 1-dimensional"): + # 0-dim + TimedeltaArray(arr[[0]].squeeze()) + def test_freq_validation(self): # ensure that the public constructor cannot create an invalid instance arr = np.array([0, 0, 1], dtype=np.int64) * 3600 * 10**9 @@ -51,6 +63,17 @@ def test_copy(self): class TestTimedeltaArray(object): + def test_np_sum(self): + # GH#25282 + # TODO: result should be a Timedelta, not a np.timedelta64 object + vals = np.arange(5).view('m8[h]').astype('m8[ns]') + arr = TimedeltaArray(vals) + result = np.sum(arr) + assert result == vals.sum() + + result = np.sum(pd.TimedeltaIndex(arr)) + assert result == vals.sum() + def test_from_sequence_dtype(self): msg = "dtype .*object.* cannot be converted to timedelta64" with pytest.raises(ValueError, match=msg): From ffacdc18cdfa35e95e7ff021e108b730967d75a7 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Fri, 15 Feb 2019 07:57:09 -0800 Subject: [PATCH 2/4] Fixes for 32 bit systems, reviewer comments --- doc/source/whatsnew/v0.24.2.rst | 3 ++- pandas/tests/arrays/test_timedeltas.py | 3 +-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/source/whatsnew/v0.24.2.rst b/doc/source/whatsnew/v0.24.2.rst index cf86c008f29e8..1ea896bcdd6d7 100644 --- a/doc/source/whatsnew/v0.24.2.rst +++ b/doc/source/whatsnew/v0.24.2.rst @@ -26,6 +26,7 @@ Fixed Regressions - Fixed regression in :meth:`DataFrame.duplicated()`, where empty dataframe was not returning a boolean dtyped Series. (:issue:`25184`) - Fixed regression in subtraction between :class:`Series` objects with ``datetime64[ns]`` dtype incorrectly raising ``OverflowError`` when the `Series` on the right contains null values (:issue:`25317`) +- Bug in :class:`TimedeltaIndex` where `np.sum(index)` incorrectly returned a zero-dimensional object instead of a scalar (:issue:`25282`) .. _whatsnew_0242.enhancements: @@ -72,7 +73,7 @@ Bug Fixes - **Timedelta** -- Bug in :class:`TimedeltaIndex` where `np.sum(index)` incorrectly returned a zero-dimensional object instead of a scalar (:issue:`25282`) + - - - diff --git a/pandas/tests/arrays/test_timedeltas.py b/pandas/tests/arrays/test_timedeltas.py index 85004b106633c..1fec533a14a6f 100644 --- a/pandas/tests/arrays/test_timedeltas.py +++ b/pandas/tests/arrays/test_timedeltas.py @@ -65,8 +65,7 @@ def test_copy(self): class TestTimedeltaArray(object): def test_np_sum(self): # GH#25282 - # TODO: result should be a Timedelta, not a np.timedelta64 object - vals = np.arange(5).view('m8[h]').astype('m8[ns]') + vals = np.arange(5, dtype=np.int64).view('m8[h]').astype('m8[ns]') arr = TimedeltaArray(vals) result = np.sum(arr) assert result == vals.sum() From 5e252972ae98bb51d5281911c021b526d14c1dfa Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Sat, 16 Feb 2019 09:38:35 -0800 Subject: [PATCH 3/4] Update doc/source/whatsnew/v0.24.2.rst Co-Authored-By: jbrockmendel --- doc/source/whatsnew/v0.24.2.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v0.24.2.rst b/doc/source/whatsnew/v0.24.2.rst index 1ea896bcdd6d7..fc42544acd9e0 100644 --- a/doc/source/whatsnew/v0.24.2.rst +++ b/doc/source/whatsnew/v0.24.2.rst @@ -26,7 +26,7 @@ Fixed Regressions - Fixed regression in :meth:`DataFrame.duplicated()`, where empty dataframe was not returning a boolean dtyped Series. (:issue:`25184`) - Fixed regression in subtraction between :class:`Series` objects with ``datetime64[ns]`` dtype incorrectly raising ``OverflowError`` when the `Series` on the right contains null values (:issue:`25317`) -- Bug in :class:`TimedeltaIndex` where `np.sum(index)` incorrectly returned a zero-dimensional object instead of a scalar (:issue:`25282`) +- Fixed regression in :class:`TimedeltaIndex` where `np.sum(index)` incorrectly returned a zero-dimensional object instead of a scalar (:issue:`25282`) .. _whatsnew_0242.enhancements: From 517bf0fe56e2356e71b7243ea9de81daf29b20b6 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Sun, 17 Feb 2019 17:59:10 -0800 Subject: [PATCH 4/4] fix array_wrap behavior on npdev --- pandas/core/indexes/base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pandas/core/indexes/base.py b/pandas/core/indexes/base.py index f2c8ac6e9b413..b5f3c929a7f36 100644 --- a/pandas/core/indexes/base.py +++ b/pandas/core/indexes/base.py @@ -665,7 +665,8 @@ def __array_wrap__(self, result, context=None): """ Gets called after a ufunc. """ - if is_bool_dtype(result): + result = lib.item_from_zerodim(result) + if is_bool_dtype(result) or lib.is_scalar(result): return result attrs = self._get_attributes_dict()