Skip to content

Commit 730c97c

Browse files
dworvosdworvos
committed
BUG: Fix merging non-indexes causes Index dtype promotion in when keys are missing from left or right side. (GH28220)
Also closes GH24897, GH24212, and GH17257
1 parent e623f0f commit 730c97c

File tree

4 files changed

+208
-2
lines changed

4 files changed

+208
-2
lines changed

Diff for: ‎doc/source/whatsnew/v0.25.2.rst

+5
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,11 @@ Groupby/resample/rolling
3434
- Bug incorrectly raising an ``IndexError`` when passing a list of quantiles to :meth:`pandas.core.groupby.DataFrameGroupBy.quantile` (:issue:`28113`).
3535
- Bug in :meth:`pandas.core.groupby.GroupBy.shift`, :meth:`pandas.core.groupby.GroupBy.bfill` and :meth:`pandas.core.groupby.GroupBy.ffill` where timezone information would be dropped (:issue:`19995`, :issue:`27992`)
3636

37+
Reshaping
38+
^^^^^^^^^
39+
40+
- Added new option to allow user to specify NA value for certain joins when missing keys when not using left_index when how='right', or right_index when how='left' causing dtype promotion (:issue:`28220`).
41+
3742
Other
3843
^^^^^
3944

Diff for: ‎pandas/core/frame.py

+6
Original file line numberDiff line numberDiff line change
@@ -231,6 +231,12 @@
231231
232232
.. versionadded:: 0.21.0
233233
234+
index_na_value : value, optional
235+
If a join requires NA values to be placed in the index use this value or
236+
accept the default NA for the dtype which may involve a type promotion
237+
238+
.. versionadded:: 0.25.2
239+
234240
Returns
235241
-------
236242
DataFrame

Diff for: ‎pandas/core/reshape/merge.py

+16-1
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,10 @@
4848
from pandas.core.sorting import is_int64_overflow_possible
4949

5050

51+
class DefaultNA:
52+
pass
53+
54+
5155
@Substitution("\nleft : DataFrame")
5256
@Appender(_merge_doc, indents=0)
5357
def merge(
@@ -64,6 +68,7 @@ def merge(
6468
copy=True,
6569
indicator=False,
6670
validate=None,
71+
index_na_value=DefaultNA(),
6772
):
6873
op = _MergeOperation(
6974
left,
@@ -79,6 +84,7 @@ def merge(
7984
copy=copy,
8085
indicator=indicator,
8186
validate=validate,
87+
index_na_value=index_na_value,
8288
)
8389
return op.get_result()
8490

@@ -551,6 +557,7 @@ def __init__(
551557
copy=True,
552558
indicator=False,
553559
validate=None,
560+
index_na_value=DefaultNA(),
554561
):
555562
left = validate_operand(left)
556563
right = validate_operand(right)
@@ -619,6 +626,10 @@ def __init__(
619626
if validate is not None:
620627
self._validate(validate)
621628

629+
# if a join requires NA values to be placed in the index
630+
# use this value or default NA which may involve a type promotion
631+
self.index_na_value = index_na_value
632+
622633
def get_result(self):
623634
if self.indicator:
624635
self.left, self.right = self._indicator_pre_merge(self.left, self.right)
@@ -898,7 +909,11 @@ def _create_join_index(
898909
# and fill_value because it throws a ValueError on integer indices
899910
mask = indexer == -1
900911
if np.any(mask):
901-
fill_value = na_value_for_dtype(index.dtype, compat=False)
912+
if isinstance(self.index_na_value, DefaultNA):
913+
fill_value = na_value_for_dtype(index.dtype, compat=False)
914+
else:
915+
fill_value = self.index_na_value
916+
902917
index = index.append(Index([fill_value]))
903918
return index.take(indexer)
904919

Diff for: ‎pandas/tests/reshape/merge/test_merge.py

+181-1
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,11 @@
2929
from pandas.core.reshape.concat import concat
3030
from pandas.core.reshape.merge import MergeError, merge
3131
import pandas.util.testing as tm
32-
from pandas.util.testing import assert_frame_equal, assert_series_equal
32+
from pandas.util.testing import (
33+
assert_frame_equal,
34+
assert_index_equal,
35+
assert_series_equal,
36+
)
3337

3438
N = 50
3539
NGROUPS = 8
@@ -2131,3 +2135,179 @@ def test_merge_multiindex_columns():
21312135
expected["id"] = ""
21322136

21332137
tm.assert_frame_equal(result, expected)
2138+
2139+
2140+
@pytest.fixture(
2141+
params=[
2142+
dict(domain=pd.Index(["A", "B", "C"])),
2143+
dict(domain=CategoricalIndex(["A", "B", "C"])),
2144+
dict(domain=DatetimeIndex(["2001-01-01", "2002-02-02", "2003-03-03"])),
2145+
dict(domain=Float64Index([1, 2, 3])),
2146+
dict(domain=Int64Index([1, 2, 3])),
2147+
dict(domain=IntervalIndex.from_tuples([(1, 2), (2, 3), (3, 4)])),
2148+
dict(domain=TimedeltaIndex(["1d", "2d", "3d"])),
2149+
dict(domain=PeriodIndex(["2001-01-01", "2001-01-02", "2001-01-03"], freq="D")),
2150+
]
2151+
)
2152+
def fix_GH_28220_(request):
2153+
class Data:
2154+
def __init__(self):
2155+
self.domain = request.param["domain"]
2156+
self.X = pd.DataFrame({"count": [1, 2]}, index=self.domain.take([0, 1]))
2157+
self.Y = pd.DataFrame(
2158+
{"name": self.domain.take([0, 2]), "value": [100, 200]}
2159+
)
2160+
self.Z = pd.DataFrame(
2161+
{"name": self.domain.take([0, 0, 2]), "value": [100, 200, 300]}
2162+
)
2163+
self.E = pd.DataFrame(columns=["name", "value"])
2164+
2165+
assert isinstance(self.X.index, type(self.domain))
2166+
2167+
return Data()
2168+
2169+
2170+
@pytest.mark.parametrize(
2171+
"how,expected",
2172+
[
2173+
("left", ([0, -255], [0, 1, -255], [0, 1])),
2174+
("inner", ([0], [0, 1], [])),
2175+
("outer", ([0, -255, 1], [0, 1, -255, 2], [0, 1])),
2176+
],
2177+
)
2178+
def test_left_index_merge_with_missing_by_right_on(fix_GH_28220_, how, expected):
2179+
2180+
# GH 28220
2181+
(e1, e2, e3) = map(lambda x: pd.Index(x), expected)
2182+
e3 = fix_GH_28220_.domain.take(e3)
2183+
2184+
r1 = pd.merge(
2185+
fix_GH_28220_.X,
2186+
fix_GH_28220_.Y,
2187+
left_index=True,
2188+
right_on=["name"],
2189+
how=how,
2190+
index_na_value=-255,
2191+
)
2192+
assert_index_equal(r1.index, e1)
2193+
2194+
r2 = pd.merge(
2195+
fix_GH_28220_.X,
2196+
fix_GH_28220_.Z,
2197+
left_index=True,
2198+
right_on=["name"],
2199+
how=how,
2200+
index_na_value=-255,
2201+
)
2202+
assert_index_equal(r2.index, e2)
2203+
2204+
r3 = pd.merge(
2205+
fix_GH_28220_.X,
2206+
fix_GH_28220_.E,
2207+
left_index=True,
2208+
right_on=["name"],
2209+
how=how,
2210+
index_na_value=-255,
2211+
)
2212+
2213+
# special case when result is empty, dtype is object
2214+
if r3.empty:
2215+
e3 = pd.Index([], dtype=object, name=e3.name)
2216+
2217+
assert_index_equal(r3.index, e3)
2218+
2219+
2220+
@pytest.mark.parametrize(
2221+
"how,expected",
2222+
[
2223+
("right", ([0, -255], [0, 0, -255], [0, 1, 2])),
2224+
("inner", ([0], [0, 0], [])),
2225+
("outer", ([0, 1, -255], [0, 0, 1, -255], [0, 1])),
2226+
],
2227+
)
2228+
def test_left_on_merge_with_missing_by_right_index(fix_GH_28220_, how, expected):
2229+
2230+
# GH 28220
2231+
(e1, e2, e3) = map(lambda x: pd.Index(x), expected)
2232+
2233+
r1 = pd.merge(
2234+
fix_GH_28220_.X.reset_index(),
2235+
fix_GH_28220_.Y.set_index("name"),
2236+
left_on=["index"],
2237+
right_index=True,
2238+
how=how,
2239+
index_na_value=-255,
2240+
)
2241+
assert_index_equal(r1.index, e1)
2242+
2243+
r2 = pd.merge(
2244+
fix_GH_28220_.X.reset_index(),
2245+
fix_GH_28220_.Z.set_index("name"),
2246+
left_on=["index"],
2247+
right_index=True,
2248+
how=how,
2249+
index_na_value=-255,
2250+
)
2251+
assert_index_equal(r2.index, e2)
2252+
2253+
r3 = pd.merge(
2254+
fix_GH_28220_.X.reset_index(),
2255+
fix_GH_28220_.E.set_index("name"),
2256+
left_on=["index"],
2257+
right_index=True,
2258+
how=how,
2259+
index_na_value=-255,
2260+
)
2261+
2262+
# special case when result is empty, dtype is object
2263+
if r3.empty:
2264+
e3 = pd.Index([], dtype=object, name=e3.name)
2265+
2266+
assert_index_equal(r3.index, e3)
2267+
2268+
2269+
@pytest.mark.parametrize(
2270+
"how,expected",
2271+
[
2272+
("left", ([0, 1], [0, 1, 2], [0, 1])),
2273+
("right", ([0, 1], [0, 1, 2], [0, 2])),
2274+
("inner", ([0], [0, 1], [])),
2275+
("outer", ([0, 1, 2], [0, 1, 2, 3], [0, 1])),
2276+
],
2277+
)
2278+
def test_left_on_merge_with_missing_by_right_on(fix_GH_28220_, how, expected):
2279+
2280+
# GH 28220
2281+
(e1, e2, e3) = map(lambda x: pd.Index(x), expected)
2282+
2283+
r1 = pd.merge(
2284+
fix_GH_28220_.X.reset_index(),
2285+
fix_GH_28220_.Y,
2286+
left_on=["index"],
2287+
right_on=["name"],
2288+
how=how,
2289+
)
2290+
assert_index_equal(r1.index, e1)
2291+
2292+
r2 = pd.merge(
2293+
fix_GH_28220_.X.reset_index(),
2294+
fix_GH_28220_.Z,
2295+
left_on=["index"],
2296+
right_on=["name"],
2297+
how=how,
2298+
)
2299+
assert_index_equal(r2.index, e2)
2300+
2301+
r3 = pd.merge(
2302+
fix_GH_28220_.X.reset_index(),
2303+
fix_GH_28220_.E,
2304+
left_on=["index"],
2305+
right_on=["name"],
2306+
how=how,
2307+
)
2308+
2309+
# special case when result is empty, dtype is object
2310+
if r3.empty:
2311+
e3 = pd.Index([], dtype=object, name=e3.name)
2312+
2313+
assert_index_equal(r3.index, e3)

0 commit comments

Comments
 (0)