From 1990e3f2431e44150c50d00abaff3e9c085603c3 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Tue, 20 Feb 2018 13:13:55 -0800 Subject: [PATCH 01/13] dispatch Series[datetime64] comparison ops to DatetimeIndex --- pandas/core/ops.py | 30 +++++++++++++++--------------- pandas/tests/test_base.py | 20 +++++++++++--------- 2 files changed, 26 insertions(+), 24 deletions(-) diff --git a/pandas/core/ops.py b/pandas/core/ops.py index da65f1f31ed2a..3fa0cc02d64fd 100644 --- a/pandas/core/ops.py +++ b/pandas/core/ops.py @@ -10,8 +10,7 @@ import numpy as np import pandas as pd -from pandas._libs import (lib, index as libindex, - algos as libalgos) +from pandas._libs import lib, algos as libalgos from pandas import compat from pandas.util._decorators import Appender @@ -944,24 +943,20 @@ def na_op(x, y): # integer comparisons # we have a datetime/timedelta and may need to convert + assert not needs_i8_conversion(x) mask = None - if (needs_i8_conversion(x) or - (not is_scalar(y) and needs_i8_conversion(y))): - - if is_scalar(y): - mask = isna(x) - y = libindex.convert_scalar(x, com._values_from_object(y)) - else: - mask = isna(x) | isna(y) - y = y.view('i8') + if not is_scalar(y) and needs_i8_conversion(y): + mask = isna(x) | isna(y) + y = y.view('i8') x = x.view('i8') - try: + method = getattr(x, name, None) + if method is not None: with np.errstate(all='ignore'): result = getattr(x, name)(y) if result is NotImplemented: raise TypeError("invalid type comparison") - except AttributeError: + else: result = op(x, y) if mask is not None and mask.any(): @@ -991,6 +986,12 @@ def wrapper(self, other, axis=None): return self._constructor(res_values, index=self.index, name=res_name) + if is_datetime64_dtype(self) or is_datetime64tz_dtype(self): + res_values = dispatch_to_index_op(op, self, other, + pd.DatetimeIndex) + return self._constructor(res_values, index=self.index, + name=res_name) + elif is_timedelta64_dtype(self): res_values = dispatch_to_index_op(op, self, other, pd.TimedeltaIndex) @@ -1008,8 +1009,7 @@ def wrapper(self, other, axis=None): elif isinstance(other, (np.ndarray, pd.Index)): # do not check length of zerodim array # as it will broadcast - if (not is_scalar(lib.item_from_zerodim(other)) and - len(self) != len(other)): + if other.ndim != 0 and len(self) != len(other): raise ValueError('Lengths must match to compare') res_values = na_op(self.values, np.asarray(other)) diff --git a/pandas/tests/test_base.py b/pandas/tests/test_base.py index 4b5ad336139b0..40400e7394ffd 100644 --- a/pandas/tests/test_base.py +++ b/pandas/tests/test_base.py @@ -10,7 +10,7 @@ import pandas as pd import pandas.compat as compat from pandas.core.dtypes.common import ( - is_object_dtype, is_datetimetz, + is_object_dtype, is_datetimetz, is_datetime64_dtype, needs_i8_conversion) import pandas.util.testing as tm from pandas import (Series, Index, DatetimeIndex, TimedeltaIndex, @@ -296,14 +296,16 @@ def test_none_comparison(self): # result = None != o # noqa # assert result.iat[0] # assert result.iat[1] - - result = None > o - assert not result.iat[0] - assert not result.iat[1] - - result = o < None - assert not result.iat[0] - assert not result.iat[1] + if not (is_datetime64_dtype(o) or is_datetimetz(o)): + # Following DatetimeIndex (and Timestamp) convention, + # inequality comparisons with Series[datetime64] raise + result = None > o + assert not result.iat[0] + assert not result.iat[1] + + result = o < None + assert not result.iat[0] + assert not result.iat[1] def test_ndarray_compat_properties(self): From f48e6d52c793dcdf8fb1092d78e7eda94be5c904 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Tue, 20 Feb 2018 14:02:26 -0800 Subject: [PATCH 02/13] remove unsupported case --- pandas/tests/indexes/datetimes/test_partial_slicing.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/tests/indexes/datetimes/test_partial_slicing.py b/pandas/tests/indexes/datetimes/test_partial_slicing.py index 6bb4229883525..f263ac78cd343 100644 --- a/pandas/tests/indexes/datetimes/test_partial_slicing.py +++ b/pandas/tests/indexes/datetimes/test_partial_slicing.py @@ -2,7 +2,7 @@ import pytest -from datetime import datetime, date +from datetime import datetime import numpy as np import pandas as pd import operator as op @@ -349,7 +349,7 @@ def test_loc_datetime_length_one(self): @pytest.mark.parametrize('datetimelike', [ Timestamp('20130101'), datetime(2013, 1, 1), - date(2013, 1, 1), np.datetime64('2013-01-01T00:00', 'ns')]) + np.datetime64('2013-01-01T00:00', 'ns')]) @pytest.mark.parametrize('op,expected', [ (op.lt, [True, False, False, False]), (op.le, [True, True, False, False]), From bd909b06002b6a4bee3b33e0b5d4913f7593ae9b Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Tue, 20 Feb 2018 15:41:31 -0800 Subject: [PATCH 03/13] fix error in older numpys --- pandas/core/indexes/datetimes.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pandas/core/indexes/datetimes.py b/pandas/core/indexes/datetimes.py index cc9ce1f3fd5eb..554e519e09beb 100644 --- a/pandas/core/indexes/datetimes.py +++ b/pandas/core/indexes/datetimes.py @@ -142,6 +142,9 @@ def wrapper(self, other): else: o_mask = other.view('i8') == libts.iNaT + # for older numpys we need to be careful not to pass a Series + # as a mask below + o_mask = com._values_from_object(o_mask) if o_mask.any(): result[o_mask] = nat_result From 003c4ff856d89fb6f03607ba522719caa393e5ce Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Tue, 20 Feb 2018 21:48:28 -0800 Subject: [PATCH 04/13] Fixup copy/paste --- pandas/core/ops.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/core/ops.py b/pandas/core/ops.py index 3fa0cc02d64fd..4363caf1aaf9e 100644 --- a/pandas/core/ops.py +++ b/pandas/core/ops.py @@ -953,7 +953,7 @@ def na_op(x, y): method = getattr(x, name, None) if method is not None: with np.errstate(all='ignore'): - result = getattr(x, name)(y) + result = method(y) if result is NotImplemented: raise TypeError("invalid type comparison") else: From f942be918738ece803b0368c08720a2f25b50afd Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Tue, 20 Feb 2018 21:57:02 -0800 Subject: [PATCH 05/13] unwrap Series to try to debug appveyor breakage --- pandas/core/indexes/base.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pandas/core/indexes/base.py b/pandas/core/indexes/base.py index 7dfa34bd634ad..68fde295c5302 100644 --- a/pandas/core/indexes/base.py +++ b/pandas/core/indexes/base.py @@ -3949,6 +3949,10 @@ def _evaluate_compare(self, other): result = _comp_method_OBJECT_ARRAY( op, self.values, other) else: + if isinstance(other, ABCSeries): + # Windows builds with some numpy versions (1.13) + # require specifically unwrapping Series GH#19800 + other = other.values with np.errstate(all='ignore'): result = op(self.values, np.asarray(other)) From 59c96e82b6617cdb0e64945fb7b95593e7475878 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Wed, 21 Feb 2018 08:52:32 -0800 Subject: [PATCH 06/13] port test for None comparison --- pandas/tests/test_base.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pandas/tests/test_base.py b/pandas/tests/test_base.py index 40400e7394ffd..6247079e4ac3a 100644 --- a/pandas/tests/test_base.py +++ b/pandas/tests/test_base.py @@ -296,9 +296,14 @@ def test_none_comparison(self): # result = None != o # noqa # assert result.iat[0] # assert result.iat[1] - if not (is_datetime64_dtype(o) or is_datetimetz(o)): + if (is_datetime64_dtype(o) or is_datetimetz(o)): # Following DatetimeIndex (and Timestamp) convention, # inequality comparisons with Series[datetime64] raise + with pytest.raises(TypeError): + None > o + with pytest.raises(TypeError): + o > None + else: result = None > o assert not result.iat[0] assert not result.iat[1] From 25534c6fd766299847737b3ccec09d95becc6abc Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Wed, 21 Feb 2018 08:59:32 -0800 Subject: [PATCH 07/13] troubleshoot appveyor build --- pandas/core/indexes/base.py | 4 ---- pandas/core/indexes/datetimes.py | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/pandas/core/indexes/base.py b/pandas/core/indexes/base.py index 68fde295c5302..7dfa34bd634ad 100644 --- a/pandas/core/indexes/base.py +++ b/pandas/core/indexes/base.py @@ -3949,10 +3949,6 @@ def _evaluate_compare(self, other): result = _comp_method_OBJECT_ARRAY( op, self.values, other) else: - if isinstance(other, ABCSeries): - # Windows builds with some numpy versions (1.13) - # require specifically unwrapping Series GH#19800 - other = other.values with np.errstate(all='ignore'): result = op(self.values, np.asarray(other)) diff --git a/pandas/core/indexes/datetimes.py b/pandas/core/indexes/datetimes.py index 554e519e09beb..a5b2bbd7f977c 100644 --- a/pandas/core/indexes/datetimes.py +++ b/pandas/core/indexes/datetimes.py @@ -137,7 +137,7 @@ def wrapper(self, other): result = func(np.asarray(other)) result = com._values_from_object(result) - if isinstance(other, Index): + if isinstance(other, (Index, ABCSeries)): o_mask = other.values.view('i8') == libts.iNaT else: o_mask = other.view('i8') == libts.iNaT From 3d159984806def30981814a2e43f6f4f99b00ad6 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Wed, 21 Feb 2018 13:45:13 -0800 Subject: [PATCH 08/13] continue troubleshooting Appveyor --- pandas/core/indexes/datetimes.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/pandas/core/indexes/datetimes.py b/pandas/core/indexes/datetimes.py index a5b2bbd7f977c..5affde78a3209 100644 --- a/pandas/core/indexes/datetimes.py +++ b/pandas/core/indexes/datetimes.py @@ -137,8 +137,17 @@ def wrapper(self, other): result = func(np.asarray(other)) result = com._values_from_object(result) - if isinstance(other, (Index, ABCSeries)): + if isinstance(other, Index): o_mask = other.values.view('i8') == libts.iNaT + elif isinstance(other, ABCSeries): + try: + # GH#19800 On 32-bit Windows builds this fails, but we can + # infer that the mask should be all-False + view = other.values.view('i8') + except ValueError: + o_mask = np.zeros(shape=other.shape, dtype=bool) + else: + o_mask = view == libts.iNaT else: o_mask = other.view('i8') == libts.iNaT From db487e8ba5f8efcdae8c949e70d0bca188c3d44e Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Wed, 21 Feb 2018 21:21:55 -0800 Subject: [PATCH 09/13] test for date comparison --- pandas/tests/series/test_arithmetic.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/pandas/tests/series/test_arithmetic.py b/pandas/tests/series/test_arithmetic.py index f727edf8fb7d8..1c723982143d2 100644 --- a/pandas/tests/series/test_arithmetic.py +++ b/pandas/tests/series/test_arithmetic.py @@ -79,6 +79,22 @@ def test_ser_cmp_result_names(self, names, op): class TestTimestampSeriesComparison(object): + def test_dt64ser_cmp_date_invalid(self): + # GH#19800 datetime.date comparison raises to + # match DatetimeIndex/Timestamp/datetime + ser = pd.Series(pd.date_range('20010101', periods=10), name='dates') + date = ser.iloc[0].to_pydatetime().date() + assert not (ser == date).any() + assert (ser != date).all() + with pytest.raises(TypeError): + ser > date + with pytest.raises(TypeError): + ser < date + with pytest.raises(TypeError): + ser >= date + with pytest.raises(TypeError): + ser <= date + def test_dt64ser_cmp_period_scalar(self): ser = Series(pd.period_range('2000-01-01', periods=10, freq='D')) val = Period('2000-01-04', freq='D') From ba290ab450f9a882ebed115f4bbac97db585051f Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Thu, 22 Feb 2018 09:04:43 -0800 Subject: [PATCH 10/13] just use isna --- pandas/core/indexes/datetimes.py | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/pandas/core/indexes/datetimes.py b/pandas/core/indexes/datetimes.py index 5affde78a3209..88b3f6155d98c 100644 --- a/pandas/core/indexes/datetimes.py +++ b/pandas/core/indexes/datetimes.py @@ -137,23 +137,7 @@ def wrapper(self, other): result = func(np.asarray(other)) result = com._values_from_object(result) - if isinstance(other, Index): - o_mask = other.values.view('i8') == libts.iNaT - elif isinstance(other, ABCSeries): - try: - # GH#19800 On 32-bit Windows builds this fails, but we can - # infer that the mask should be all-False - view = other.values.view('i8') - except ValueError: - o_mask = np.zeros(shape=other.shape, dtype=bool) - else: - o_mask = view == libts.iNaT - else: - o_mask = other.view('i8') == libts.iNaT - - # for older numpys we need to be careful not to pass a Series - # as a mask below - o_mask = com._values_from_object(o_mask) + o_mask = np.array(isna(other)) if o_mask.any(): result[o_mask] = nat_result From f8d5bddabdce0d90ef10ddfa9de1013138a4ceff Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Sat, 24 Feb 2018 08:42:50 -0800 Subject: [PATCH 11/13] comments --- pandas/core/indexes/datetimes.py | 2 ++ pandas/core/ops.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/pandas/core/indexes/datetimes.py b/pandas/core/indexes/datetimes.py index 88b3f6155d98c..8d70ef7a30036 100644 --- a/pandas/core/indexes/datetimes.py +++ b/pandas/core/indexes/datetimes.py @@ -137,6 +137,8 @@ def wrapper(self, other): result = func(np.asarray(other)) result = com._values_from_object(result) + # Make sure to pass an array to result[...]; indexing with + # Series breaks with older version of numpy o_mask = np.array(isna(other)) if o_mask.any(): result[o_mask] = nat_result diff --git a/pandas/core/ops.py b/pandas/core/ops.py index 4363caf1aaf9e..08837f386aac9 100644 --- a/pandas/core/ops.py +++ b/pandas/core/ops.py @@ -987,6 +987,8 @@ def wrapper(self, other, axis=None): name=res_name) if is_datetime64_dtype(self) or is_datetime64tz_dtype(self): + # Dispatch to DatetimeIndex to ensure identical + # Series/Index behavior res_values = dispatch_to_index_op(op, self, other, pd.DatetimeIndex) return self._constructor(res_values, index=self.index, From b618a827d5a77e3268ed58e6778de6edc72721f2 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Sat, 24 Feb 2018 11:05:46 -0800 Subject: [PATCH 12/13] comment --- pandas/tests/series/test_arithmetic.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pandas/tests/series/test_arithmetic.py b/pandas/tests/series/test_arithmetic.py index 057b5db92c96d..ec0d7296e540e 100644 --- a/pandas/tests/series/test_arithmetic.py +++ b/pandas/tests/series/test_arithmetic.py @@ -90,7 +90,8 @@ def test_ser_cmp_result_names(self, names, op): class TestTimestampSeriesComparison(object): def test_dt64ser_cmp_date_invalid(self): # GH#19800 datetime.date comparison raises to - # match DatetimeIndex/Timestamp/datetime + # match DatetimeIndex/Timestamp. This also matches the behavior + # of stdlib datetime.datetime ser = pd.Series(pd.date_range('20010101', periods=10), name='dates') date = ser.iloc[0].to_pydatetime().date() assert not (ser == date).any() From ff516344f83bea946c2743810bea459f3182a265 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Wed, 28 Feb 2018 20:04:31 -0800 Subject: [PATCH 13/13] Update ops.py --- pandas/core/ops.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/core/ops.py b/pandas/core/ops.py index 0ded8556c924d..931e91b941a7e 100644 --- a/pandas/core/ops.py +++ b/pandas/core/ops.py @@ -10,7 +10,7 @@ import numpy as np import pandas as pd -from pandas._libs import lib, algos as libalgos, ops as libops +from pandas._libs import algos as libalgos, ops as libops from pandas import compat from pandas.util._decorators import Appender