From 8cb06545fd4a9e0947319da30e549392bc3bb2a3 Mon Sep 17 00:00:00 2001 From: fjdiod Date: Sun, 29 Jul 2018 00:05:09 +0300 Subject: [PATCH 1/7] BUG: astype with timedelta and datetime string (#22100) --- pandas/core/dtypes/cast.py | 2 +- pandas/tests/dtypes/test_cast.py | 14 +++++++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/pandas/core/dtypes/cast.py b/pandas/core/dtypes/cast.py index ead7b39309f5e..fea7f2e31b44c 100644 --- a/pandas/core/dtypes/cast.py +++ b/pandas/core/dtypes/cast.py @@ -712,7 +712,7 @@ def astype_nansafe(arr, dtype, copy=True): elif is_object_dtype(arr): # work around NumPy brokenness, #1987 - if np.issubdtype(dtype.type, np.integer): + if np.issubdtype(dtype.type, np.integer) and not is_datetime64_dtype(dtype) and not is_timedelta64_dtype(dtype): return lib.astype_intsafe(arr.ravel(), dtype).reshape(arr.shape) # if we have a datetime/timedelta array of objects diff --git a/pandas/tests/dtypes/test_cast.py b/pandas/tests/dtypes/test_cast.py index 0d6382424ccf5..928d7e138adcc 100644 --- a/pandas/tests/dtypes/test_cast.py +++ b/pandas/tests/dtypes/test_cast.py @@ -24,7 +24,8 @@ find_common_type, construct_1d_object_array_from_listlike, construct_1d_ndarray_preserving_na, - construct_1d_arraylike_from_scalar) + construct_1d_arraylike_from_scalar, + astype_nansafe) from pandas.core.dtypes.dtypes import ( CategoricalDtype, DatetimeTZDtype, @@ -456,3 +457,14 @@ def test_cast_1d_arraylike_from_scalar_categorical(self): def test_construct_1d_ndarray_preserving_na(values, dtype, expected): result = construct_1d_ndarray_preserving_na(values, dtype=dtype) tm.assert_numpy_array_equal(result, expected) + +@pytest.mark.parametrize('arr, dtype, expected', [ + (np.array(['0:0:1'], dtype='object'), 'timedelta64[ns]', 'timedelta64[ns]'), + (np.array(['0:0:1'], dtype='object'), 'timedelta64', 'timedelta64'), + (np.array(['2000'], dtype='object'), 'datetime64[ns]', 'datetime64[ns]'), + (np.array(['2000'], dtype='object'), 'datetime64', 'datetime64'), +]) +def test_astype_nansafe(arr, dtype, expected): + # GH #22100 + result = astype_nansafe(arr, dtype) + is_dtype_equal(arr.dtype, expected) From 52257e329cd94cb8a1b2843d9366aae4cd100aa9 Mon Sep 17 00:00:00 2001 From: fjdiod Date: Sun, 29 Jul 2018 00:24:21 +0300 Subject: [PATCH 2/7] fix style --- pandas/core/dtypes/cast.py | 3 ++- pandas/tests/dtypes/test_cast.py | 15 ++++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/pandas/core/dtypes/cast.py b/pandas/core/dtypes/cast.py index fea7f2e31b44c..0a9176cd231a3 100644 --- a/pandas/core/dtypes/cast.py +++ b/pandas/core/dtypes/cast.py @@ -712,7 +712,8 @@ def astype_nansafe(arr, dtype, copy=True): elif is_object_dtype(arr): # work around NumPy brokenness, #1987 - if np.issubdtype(dtype.type, np.integer) and not is_datetime64_dtype(dtype) and not is_timedelta64_dtype(dtype): + is_time = is_datetime64_dtype(dtype) or is_timedelta64_dtype(dtype) + if np.issubdtype(dtype.type, np.integer) and not is_time: return lib.astype_intsafe(arr.ravel(), dtype).reshape(arr.shape) # if we have a datetime/timedelta array of objects diff --git a/pandas/tests/dtypes/test_cast.py b/pandas/tests/dtypes/test_cast.py index 928d7e138adcc..1cebdf3e023f7 100644 --- a/pandas/tests/dtypes/test_cast.py +++ b/pandas/tests/dtypes/test_cast.py @@ -458,13 +458,14 @@ def test_construct_1d_ndarray_preserving_na(values, dtype, expected): result = construct_1d_ndarray_preserving_na(values, dtype=dtype) tm.assert_numpy_array_equal(result, expected) -@pytest.mark.parametrize('arr, dtype, expected', [ - (np.array(['0:0:1'], dtype='object'), 'timedelta64[ns]', 'timedelta64[ns]'), - (np.array(['0:0:1'], dtype='object'), 'timedelta64', 'timedelta64'), - (np.array(['2000'], dtype='object'), 'datetime64[ns]', 'datetime64[ns]'), - (np.array(['2000'], dtype='object'), 'datetime64', 'datetime64'), + +@pytest.mark.parametrize('arr, dtype', [ + (np.array(['0:0:1'], dtype='object'), 'timedelta64[ns]'), + (np.array(['0:0:1'], dtype='object'), 'timedelta64'), + (np.array(['2000'], dtype='object'), 'datetime64[ns]'), + (np.array(['2000'], dtype='object'), 'datetime64'), ]) -def test_astype_nansafe(arr, dtype, expected): +def test_astype_nansafe(arr, dtype): # GH #22100 result = astype_nansafe(arr, dtype) - is_dtype_equal(arr.dtype, expected) + is_dtype_equal(result.dtype, dtype) From 7196cf55ab3e9a9a6bf98740a5a5fa1fb79e17f0 Mon Sep 17 00:00:00 2001 From: fjdiod Date: Sun, 29 Jul 2018 01:12:18 +0300 Subject: [PATCH 3/7] fix test --- pandas/tests/dtypes/test_cast.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/pandas/tests/dtypes/test_cast.py b/pandas/tests/dtypes/test_cast.py index 1cebdf3e023f7..502bda9a3ea83 100644 --- a/pandas/tests/dtypes/test_cast.py +++ b/pandas/tests/dtypes/test_cast.py @@ -459,13 +459,17 @@ def test_construct_1d_ndarray_preserving_na(values, dtype, expected): tm.assert_numpy_array_equal(result, expected) -@pytest.mark.parametrize('arr, dtype', [ - (np.array(['0:0:1'], dtype='object'), 'timedelta64[ns]'), - (np.array(['0:0:1'], dtype='object'), 'timedelta64'), - (np.array(['2000'], dtype='object'), 'datetime64[ns]'), - (np.array(['2000'], dtype='object'), 'datetime64'), +@pytest.mark.parametrize('arr, dtype, expected', [ + (np.array(['0:0:1'], dtype='object'), + 'timedelta64[ns]', 'timedelta64[ns]'), + (np.array(['0:0:1'], dtype='object'), + 'timedelta64', 'float64'), + (np.array(['2000'], dtype='object'), + 'datetime64[ns]', 'datetime64[ns]'), + (np.array(['2000'], dtype='object'), + 'datetime64', 'datetime64[ns]'), ]) -def test_astype_nansafe(arr, dtype): +def test_astype_nansafe(arr, dtype, expected): # GH #22100 result = astype_nansafe(arr, dtype) - is_dtype_equal(result.dtype, dtype) + assert is_dtype_equal(result.dtype, expected) From e508d936379010a282af641fd97da47f266a322f Mon Sep 17 00:00:00 2001 From: fjdiod Date: Sun, 29 Jul 2018 03:34:46 +0300 Subject: [PATCH 4/7] fix m8 --- pandas/core/dtypes/cast.py | 2 +- pandas/tests/dtypes/test_cast.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/pandas/core/dtypes/cast.py b/pandas/core/dtypes/cast.py index 0a9176cd231a3..206476ced6bcd 100644 --- a/pandas/core/dtypes/cast.py +++ b/pandas/core/dtypes/cast.py @@ -712,7 +712,7 @@ def astype_nansafe(arr, dtype, copy=True): elif is_object_dtype(arr): # work around NumPy brokenness, #1987 - is_time = is_datetime64_dtype(dtype) or is_timedelta64_dtype(dtype) + is_time = is_datetime64_dtype(dtype) or dtype == _TD_DTYPE if np.issubdtype(dtype.type, np.integer) and not is_time: return lib.astype_intsafe(arr.ravel(), dtype).reshape(arr.shape) diff --git a/pandas/tests/dtypes/test_cast.py b/pandas/tests/dtypes/test_cast.py index 502bda9a3ea83..17cd1e600f12e 100644 --- a/pandas/tests/dtypes/test_cast.py +++ b/pandas/tests/dtypes/test_cast.py @@ -462,8 +462,6 @@ def test_construct_1d_ndarray_preserving_na(values, dtype, expected): @pytest.mark.parametrize('arr, dtype, expected', [ (np.array(['0:0:1'], dtype='object'), 'timedelta64[ns]', 'timedelta64[ns]'), - (np.array(['0:0:1'], dtype='object'), - 'timedelta64', 'float64'), (np.array(['2000'], dtype='object'), 'datetime64[ns]', 'datetime64[ns]'), (np.array(['2000'], dtype='object'), From 7be8de9ba2e51f035dad95f6fd816c32f83abc57 Mon Sep 17 00:00:00 2001 From: Sergey Solovev Date: Mon, 30 Jul 2018 15:51:46 +0300 Subject: [PATCH 5/7] add whatsnew entry --- doc/source/whatsnew/v0.24.0.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v0.24.0.txt b/doc/source/whatsnew/v0.24.0.txt index d2d5d40393b62..c86fee9b4c523 100644 --- a/doc/source/whatsnew/v0.24.0.txt +++ b/doc/source/whatsnew/v0.24.0.txt @@ -500,7 +500,7 @@ Datetimelike Timedelta ^^^^^^^^^ -- +- Fixed bug where :meth:`DataFrame.astype` could not convert timedelta and datetime strings (:issue:`#22100`) - - From 21c0520904e87b2f174fbc7c11258a1225b6ed80 Mon Sep 17 00:00:00 2001 From: Sergey Solovev Date: Tue, 31 Jul 2018 11:43:14 +0300 Subject: [PATCH 6/7] fix test_astype_nansafe --- pandas/tests/dtypes/test_cast.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/pandas/tests/dtypes/test_cast.py b/pandas/tests/dtypes/test_cast.py index 17cd1e600f12e..438a4a2dfa56f 100644 --- a/pandas/tests/dtypes/test_cast.py +++ b/pandas/tests/dtypes/test_cast.py @@ -460,12 +460,9 @@ def test_construct_1d_ndarray_preserving_na(values, dtype, expected): @pytest.mark.parametrize('arr, dtype, expected', [ - (np.array(['0:0:1'], dtype='object'), - 'timedelta64[ns]', 'timedelta64[ns]'), - (np.array(['2000'], dtype='object'), - 'datetime64[ns]', 'datetime64[ns]'), - (np.array(['2000'], dtype='object'), - 'datetime64', 'datetime64[ns]'), + (np.array(['0:0:1'], dtype='O'), 'm8[ns]', 'm8[ns]'), + (np.array(['2000'], dtype='O'), 'M8[ns]', 'M8[ns]'), + (np.array(['2000'], dtype='O'), 'M8', 'M8[ns]'), ]) def test_astype_nansafe(arr, dtype, expected): # GH #22100 From 74e670b6bc616055fe03e3f9e72c4560d03187a5 Mon Sep 17 00:00:00 2001 From: Sergey Solovev Date: Tue, 31 Jul 2018 13:45:18 +0300 Subject: [PATCH 7/7] fix datetime64 in astype_nansafe --- doc/source/whatsnew/v0.24.0.txt | 2 +- pandas/core/dtypes/cast.py | 4 ++-- pandas/tests/dtypes/test_cast.py | 3 +-- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/doc/source/whatsnew/v0.24.0.txt b/doc/source/whatsnew/v0.24.0.txt index c86fee9b4c523..64dc9ea51eadd 100644 --- a/doc/source/whatsnew/v0.24.0.txt +++ b/doc/source/whatsnew/v0.24.0.txt @@ -500,7 +500,7 @@ Datetimelike Timedelta ^^^^^^^^^ -- Fixed bug where :meth:`DataFrame.astype` could not convert timedelta and datetime strings (:issue:`#22100`) +- Fixed bug where :meth:`DataFrame.astype` could not convert timedelta strings (:issue:`#22100`) - - diff --git a/pandas/core/dtypes/cast.py b/pandas/core/dtypes/cast.py index 294b14630423e..3f6f0bc4a8ff1 100644 --- a/pandas/core/dtypes/cast.py +++ b/pandas/core/dtypes/cast.py @@ -712,8 +712,8 @@ def astype_nansafe(arr, dtype, copy=True): elif is_object_dtype(arr): # work around NumPy brokenness, #1987 - is_time = is_datetime64_dtype(dtype) or dtype == _TD_DTYPE - if np.issubdtype(dtype.type, np.integer) and not is_time: + is_time = is_timedelta64_dtype(dtype) + if np.issubdtype(dtype.type, np.integer) and is_time: return lib.astype_intsafe(arr.ravel(), dtype).reshape(arr.shape) # if we have a datetime/timedelta array of objects diff --git a/pandas/tests/dtypes/test_cast.py b/pandas/tests/dtypes/test_cast.py index 438a4a2dfa56f..8c8e6c9a4c242 100644 --- a/pandas/tests/dtypes/test_cast.py +++ b/pandas/tests/dtypes/test_cast.py @@ -461,8 +461,7 @@ def test_construct_1d_ndarray_preserving_na(values, dtype, expected): @pytest.mark.parametrize('arr, dtype, expected', [ (np.array(['0:0:1'], dtype='O'), 'm8[ns]', 'm8[ns]'), - (np.array(['2000'], dtype='O'), 'M8[ns]', 'M8[ns]'), - (np.array(['2000'], dtype='O'), 'M8', 'M8[ns]'), + (np.array(['0:0:1'], dtype='O'), 'm8', 'float64'), ]) def test_astype_nansafe(arr, dtype, expected): # GH #22100