Skip to content

Commit 2d62018

Browse files
committed
Merge remote-tracking branch 'upstream/master' into Rt05
* upstream/master: DOC: CategoricalIndex doc string (pandas-dev#24852) CI: add __init__.py to isort skip list (pandas-dev#25455) TST: numpy RuntimeWarning with Series.round() (pandas-dev#25432) DOC: fixed geo accessor example in extending.rst (pandas-dev#25420) BUG: fixed merging with empty frame containing an Int64 column (pandas-dev#25183) (pandas-dev#25289) TST: remove never-used singleton fixtures (pandas-dev#24885) PERF/REF: improve performance of Series.searchsorted, PandasArray.searchsorted, collect functionality (pandas-dev#22034) BUG: Indexing with UTC offset string no longer ignored (pandas-dev#25263) API/ERR: allow iterators in df.set_index & improve errors (pandas-dev#24984) DOC: Rewriting of ParserError doc + minor spacing (pandas-dev#25421) ENH: Add in sort keyword to DatetimeIndex.union (pandas-dev#25110) ERR: doc update for ParsingError (pandas-dev#25414) BUG: Fix type coercion in read_json orient='table' (pandas-dev#21345) (pandas-dev#25219) DEP: add pytest-mock to environment.yml (pandas-dev#25417) Correct a typo of version number for interpolate() (pandas-dev#25418) Mark test_pct_max_many_rows as high memory (pandas-dev#25400) DOC: Edited docstring of Interval (pandas-dev#25410)
2 parents b847e4e + c986386 commit 2d62018

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

53 files changed

+765
-216
lines changed

asv_bench/benchmarks/series_methods.py

+19
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,25 @@ def time_dropna(self, dtype):
124124
self.s.dropna()
125125

126126

127+
class SearchSorted(object):
128+
129+
goal_time = 0.2
130+
params = ['int8', 'int16', 'int32', 'int64',
131+
'uint8', 'uint16', 'uint32', 'uint64',
132+
'float16', 'float32', 'float64',
133+
'str']
134+
param_names = ['dtype']
135+
136+
def setup(self, dtype):
137+
N = 10**5
138+
data = np.array([1] * N + [2] * N + [3] * N).astype(dtype)
139+
self.s = Series(data)
140+
141+
def time_searchsorted(self, dtype):
142+
key = '2' if dtype == 'str' else 2
143+
self.s.searchsorted(key)
144+
145+
127146
class Map(object):
128147

129148
params = ['dict', 'Series']

ci/deps/azure-27-compat.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ dependencies:
2121
- pytest
2222
- pytest-xdist
2323
- pytest-mock
24+
- isort
2425
- pip:
2526
- html5lib==1.0b2
2627
- beautifulsoup4==4.2.1

ci/deps/azure-27-locale.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ dependencies:
2424
- pytest-xdist
2525
- pytest-mock
2626
- hypothesis>=3.58.0
27+
- isort
2728
- pip:
2829
- html5lib==1.0b2
2930
- beautifulsoup4==4.2.1

ci/deps/azure-36-locale_slow.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -30,5 +30,6 @@ dependencies:
3030
- pytest-xdist
3131
- pytest-mock
3232
- moto
33+
- isort
3334
- pip:
3435
- hypothesis>=3.58.0

ci/deps/azure-37-locale.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ dependencies:
2828
- pytest
2929
- pytest-xdist
3030
- pytest-mock
31+
- isort
3132
- pip:
3233
- hypothesis>=3.58.0
3334
- moto # latest moto in conda-forge fails with 3.7, move to conda dependencies when this is fixed

ci/deps/azure-37-numpydev.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ dependencies:
1010
- pytest-xdist
1111
- pytest-mock
1212
- hypothesis>=3.58.0
13+
- isort
1314
- pip:
1415
- "git+git://github.com/dateutil/dateutil.git"
1516
- "-f https://7933911d6844c6c53a7d-47bd50c35cd79bd838daf386af554a83.ssl.cf2.rackcdn.com"

ci/deps/azure-macos-35.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ dependencies:
2525
- pytest
2626
- pytest-xdist
2727
- pytest-mock
28+
- isort
2829
- pip:
2930
- python-dateutil==2.5.3
3031
- hypothesis>=3.58.0

ci/deps/azure-windows-27.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -30,3 +30,4 @@ dependencies:
3030
- pytest-mock
3131
- moto
3232
- hypothesis>=3.58.0
33+
- isort

ci/deps/azure-windows-36.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -27,3 +27,4 @@ dependencies:
2727
- pytest-xdist
2828
- pytest-mock
2929
- hypothesis>=3.58.0
30+
- isort

ci/deps/travis-27.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ dependencies:
4444
- pytest-mock
4545
- moto==1.3.4
4646
- hypothesis>=3.58.0
47+
- isort
4748
- pip:
4849
- backports.lzma
4950
- pandas-gbq

ci/deps/travis-36-doc.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -43,3 +43,4 @@ dependencies:
4343
# universal
4444
- pytest
4545
- pytest-xdist
46+
- isort

ci/deps/travis-36-locale.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -32,5 +32,6 @@ dependencies:
3232
- pytest-xdist
3333
- pytest-mock
3434
- moto
35+
- isort
3536
- pip:
3637
- hypothesis>=3.58.0

ci/deps/travis-36-slow.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -30,3 +30,4 @@ dependencies:
3030
- pytest-mock
3131
- moto
3232
- hypothesis>=3.58.0
33+
- isort

ci/deps/travis-36.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ dependencies:
3838
- pytest-cov
3939
- pytest-mock
4040
- hypothesis>=3.58.0
41+
- isort
4142
- pip:
4243
- brotlipy
4344
- coverage

ci/deps/travis-37.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -17,5 +17,6 @@ dependencies:
1717
- pytest-mock
1818
- hypothesis>=3.58.0
1919
- s3fs
20+
- isort
2021
- pip:
2122
- moto

doc/source/development/extending.rst

+3-2
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,9 @@ decorate a class, providing the name of attribute to add. The class's
3333
3434
@staticmethod
3535
def _validate(obj):
36-
if 'lat' not in obj.columns or 'lon' not in obj.columns:
37-
raise AttributeError("Must have 'lat' and 'lon'.")
36+
# verify there is a column latitude and a column longitude
37+
if 'latitude' not in obj.columns or 'longitude' not in obj.columns:
38+
raise AttributeError("Must have 'latitude' and 'longitude'.")
3839
3940
@property
4041
def center(self):

doc/source/styled.xlsx

5.55 KB
Binary file not shown.

doc/source/user_guide/missing_data.rst

+1-1
Original file line numberDiff line numberDiff line change
@@ -335,7 +335,7 @@ examined :ref:`in the API <api.dataframe.missing>`.
335335
Interpolation
336336
~~~~~~~~~~~~~
337337

338-
.. versionadded:: 0.21.0
338+
.. versionadded:: 0.23.0
339339

340340
The ``limit_area`` keyword argument was added.
341341

doc/source/user_guide/timeseries.rst

+10
Original file line numberDiff line numberDiff line change
@@ -633,6 +633,16 @@ We are stopping on the included end-point as it is part of the index:
633633
dft2 = dft2.swaplevel(0, 1).sort_index()
634634
dft2.loc[idx[:, '2013-01-05'], :]
635635
636+
.. versionadded:: 0.25.0
637+
638+
Slicing with string indexing also honors UTC offset.
639+
640+
.. ipython:: python
641+
642+
df = pd.DataFrame([0], index=pd.DatetimeIndex(['2019-01-01'], tz='US/Pacific'))
643+
df
644+
df['2019-01-01 12:00:00+04:00':'2019-01-01 13:00:00+04:00']
645+
636646
.. _timeseries.slice_vs_exact_match:
637647

638648
Slice vs. Exact Match

doc/source/whatsnew/v0.24.2.rst

+1-1
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,7 @@ Bug Fixes
9696
**Other**
9797

9898
- Bug in :meth:`Series.is_unique` where single occurrences of ``NaN`` were not considered unique (:issue:`25180`)
99-
-
99+
- Bug in :func:`merge` when merging an empty ``DataFrame`` with an ``Int64`` column or a non-empty ``DataFrame`` with an ``Int64`` column that is all ``NaN`` (:issue:`25183`)
100100
-
101101

102102
.. _whatsnew_0.242.contributors:

doc/source/whatsnew/v0.25.0.rst

+37-3
Original file line numberDiff line numberDiff line change
@@ -22,14 +22,46 @@ Other Enhancements
2222
- Indexing of ``DataFrame`` and ``Series`` now accepts zerodim ``np.ndarray`` (:issue:`24919`)
2323
- :meth:`Timestamp.replace` now supports the ``fold`` argument to disambiguate DST transition times (:issue:`25017`)
2424
- :meth:`DataFrame.at_time` and :meth:`Series.at_time` now support :meth:`datetime.time` objects with timezones (:issue:`24043`)
25+
- :meth:`DataFrame.set_index` now works for instances of ``abc.Iterator``, provided their output is of the same length as the calling frame (:issue:`22484`, :issue:`24984`)
26+
- :meth:`DatetimeIndex.union` now supports the ``sort`` argument. The behaviour of the sort parameter matches that of :meth:`Index.union` (:issue:`24994`)
2527
-
2628

2729
.. _whatsnew_0250.api_breaking:
2830

2931
Backwards incompatible API changes
3032
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
3133

32-
- :meth:`Timestamp.strptime` will now raise a NotImplementedError (:issue:`25016`)
34+
.. _whatsnew_0250.api_breaking.utc_offset_indexing:
35+
36+
Indexing with date strings with UTC offsets
37+
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
38+
39+
Indexing a :class:`DataFrame` or :class:`Series` with a :class:`DatetimeIndex` with a
40+
date string with a UTC offset would previously ignore the UTC offset. Now, the UTC offset
41+
is respected in indexing. (:issue:`24076`, :issue:`16785`)
42+
43+
*Previous Behavior*:
44+
45+
.. code-block:: ipython
46+
47+
In [1]: df = pd.DataFrame([0], index=pd.DatetimeIndex(['2019-01-01'], tz='US/Pacific'))
48+
49+
In [2]: df
50+
Out[2]:
51+
0
52+
2019-01-01 00:00:00-08:00 0
53+
54+
In [3]: df['2019-01-01 00:00:00+04:00':'2019-01-01 01:00:00+04:00']
55+
Out[3]:
56+
0
57+
2019-01-01 00:00:00-08:00 0
58+
59+
*New Behavior*:
60+
61+
.. ipython:: ipython
62+
63+
df = pd.DataFrame([0], index=pd.DatetimeIndex(['2019-01-01'], tz='US/Pacific'))
64+
df['2019-01-01 12:00:00+04:00':'2019-01-01 13:00:00+04:00']
3365

3466
.. _whatsnew_0250.api.other:
3567

@@ -38,7 +70,7 @@ Other API Changes
3870

3971
- :class:`DatetimeTZDtype` will now standardize pytz timezones to a common timezone instance (:issue:`24713`)
4072
- ``Timestamp`` and ``Timedelta`` scalars now implement the :meth:`to_numpy` method as aliases to :meth:`Timestamp.to_datetime64` and :meth:`Timedelta.to_timedelta64`, respectively. (:issue:`24653`)
41-
-
73+
- :meth:`Timestamp.strptime` will now rise a ``NotImplementedError`` (:issue:`25016`)
4274
-
4375

4476
.. _whatsnew_0250.deprecations:
@@ -64,7 +96,8 @@ Performance Improvements
6496

6597
- Significant speedup in `SparseArray` initialization that benefits most operations, fixing performance regression introduced in v0.20.0 (:issue:`24985`)
6698
- `DataFrame.to_stata()` is now faster when outputting data with any string or non-native endian columns (:issue:`25045`)
67-
-
99+
- Improved performance of :meth:`Series.searchsorted`. The speedup is especially large when the dtype is
100+
int8/int16/int32 and the searched key is within the integer bounds for the dtype (:issue:`22034`)
68101

69102

70103
.. _whatsnew_0250.bug_fixes:
@@ -160,6 +193,7 @@ I/O
160193
^^^
161194

162195
- Fixed bug in missing text when using :meth:`to_clipboard` if copying utf-16 characters in Python 3 on Windows (:issue:`25040`)
196+
- Bug in :func:`read_json` for ``orient='table'`` when it tries to infer dtypes by default, which is not applicable as dtypes are already defined in the JSON schema (:issue:`21345`)
163197
-
164198
-
165199
-

environment.yml

+1
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ dependencies:
2020
- isort
2121
- moto
2222
- pytest>=4.0
23+
- pytest-mock
2324
- sphinx
2425
- numpydoc
2526

pandas/_libs/interval.pyx

-3
Original file line numberDiff line numberDiff line change
@@ -150,9 +150,6 @@ cdef class Interval(IntervalMixin):
150150
Left bound for the interval.
151151
right : orderable scalar
152152
Right bound for the interval.
153-
closed : {'left', 'right', 'both', 'neither'}, default 'right'
154-
Whether the interval is closed on the left-side, right-side, both or
155-
neither.
156153
closed : {'right', 'left', 'both', 'neither'}, default 'right'
157154
Whether the interval is closed on the left-side, right-side, both or
158155
neither. See the Notes for more detailed explanation.

pandas/compat/__init__.py

+2
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,7 @@ def lfilter(*args, **kwargs):
137137
reload = reload
138138
Hashable = collections.abc.Hashable
139139
Iterable = collections.abc.Iterable
140+
Iterator = collections.abc.Iterator
140141
Mapping = collections.abc.Mapping
141142
MutableMapping = collections.abc.MutableMapping
142143
Sequence = collections.abc.Sequence
@@ -199,6 +200,7 @@ def get_range_parameters(data):
199200

200201
Hashable = collections.Hashable
201202
Iterable = collections.Iterable
203+
Iterator = collections.Iterator
202204
Mapping = collections.Mapping
203205
MutableMapping = collections.MutableMapping
204206
Sequence = collections.Sequence

pandas/core/algorithms.py

+84-1
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
ensure_float64, ensure_int64, ensure_object, ensure_platform_int,
2020
ensure_uint64, is_array_like, is_bool_dtype, is_categorical_dtype,
2121
is_complex_dtype, is_datetime64_any_dtype, is_datetime64tz_dtype,
22-
is_datetimelike, is_extension_array_dtype, is_float_dtype,
22+
is_datetimelike, is_extension_array_dtype, is_float_dtype, is_integer,
2323
is_integer_dtype, is_interval_dtype, is_list_like, is_numeric_dtype,
2424
is_object_dtype, is_period_dtype, is_scalar, is_signed_integer_dtype,
2525
is_sparse, is_timedelta64_dtype, is_unsigned_integer_dtype,
@@ -1729,6 +1729,89 @@ def func(arr, indexer, out, fill_value=np.nan):
17291729
return out
17301730

17311731

1732+
# ------------ #
1733+
# searchsorted #
1734+
# ------------ #
1735+
1736+
def searchsorted(arr, value, side="left", sorter=None):
1737+
"""
1738+
Find indices where elements should be inserted to maintain order.
1739+
1740+
.. versionadded:: 0.25.0
1741+
1742+
Find the indices into a sorted array `arr` (a) such that, if the
1743+
corresponding elements in `value` were inserted before the indices,
1744+
the order of `arr` would be preserved.
1745+
1746+
Assuming that `arr` is sorted:
1747+
1748+
====== ================================
1749+
`side` returned index `i` satisfies
1750+
====== ================================
1751+
left ``arr[i-1] < value <= self[i]``
1752+
right ``arr[i-1] <= value < self[i]``
1753+
====== ================================
1754+
1755+
Parameters
1756+
----------
1757+
arr: array-like
1758+
Input array. If `sorter` is None, then it must be sorted in
1759+
ascending order, otherwise `sorter` must be an array of indices
1760+
that sort it.
1761+
value : array_like
1762+
Values to insert into `arr`.
1763+
side : {'left', 'right'}, optional
1764+
If 'left', the index of the first suitable location found is given.
1765+
If 'right', return the last such index. If there is no suitable
1766+
index, return either 0 or N (where N is the length of `self`).
1767+
sorter : 1-D array_like, optional
1768+
Optional array of integer indices that sort array a into ascending
1769+
order. They are typically the result of argsort.
1770+
1771+
Returns
1772+
-------
1773+
array of ints
1774+
Array of insertion points with the same shape as `value`.
1775+
1776+
See Also
1777+
--------
1778+
numpy.searchsorted : Similar method from NumPy.
1779+
"""
1780+
if sorter is not None:
1781+
sorter = ensure_platform_int(sorter)
1782+
1783+
if isinstance(arr, np.ndarray) and is_integer_dtype(arr) and (
1784+
is_integer(value) or is_integer_dtype(value)):
1785+
from .arrays.array_ import array
1786+
# if `arr` and `value` have different dtypes, `arr` would be
1787+
# recast by numpy, causing a slow search.
1788+
# Before searching below, we therefore try to give `value` the
1789+
# same dtype as `arr`, while guarding against integer overflows.
1790+
iinfo = np.iinfo(arr.dtype.type)
1791+
value_arr = np.array([value]) if is_scalar(value) else np.array(value)
1792+
if (value_arr >= iinfo.min).all() and (value_arr <= iinfo.max).all():
1793+
# value within bounds, so no overflow, so can convert value dtype
1794+
# to dtype of arr
1795+
dtype = arr.dtype
1796+
else:
1797+
dtype = value_arr.dtype
1798+
1799+
if is_scalar(value):
1800+
value = dtype.type(value)
1801+
else:
1802+
value = array(value, dtype=dtype)
1803+
elif not (is_object_dtype(arr) or is_numeric_dtype(arr) or
1804+
is_categorical_dtype(arr)):
1805+
from pandas.core.series import Series
1806+
# E.g. if `arr` is an array with dtype='datetime64[ns]'
1807+
# and `value` is a pd.Timestamp, we may need to convert value
1808+
value_ser = Series(value)._values
1809+
value = value_ser[0] if is_scalar(value) else value_ser
1810+
1811+
result = arr.searchsorted(value, side=side, sorter=sorter)
1812+
return result
1813+
1814+
17321815
# ---- #
17331816
# diff #
17341817
# ---- #

0 commit comments

Comments
 (0)