Backport PR pandas-dev#48702 on branch 1.5.x (REGR: dropna affects observed in groupby) (pandas-dev#48750)

Backport PR pandas-dev#48702: REGR: dropna affects observed in groupby

Co-authored-by: Richard Shadrach <[email protected]>

Author: meeseeksmachine

doc/source/whatsnew/v1.5.1.rst

@@ -10,6 +10,61 @@ including other versions of pandas.
 
 .. ---------------------------------------------------------------------------
 
+.. _whatsnew_151.groupby_categorical_regr:
+
+Behavior of ``groupby`` with categorical groupers (:issue:`48645`)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+In versions of pandas prior to 1.5, ``groupby`` with ``dropna=False`` would still drop
+NA values when the grouper was a categorical dtype. A fix for this was attempted in
+1.5, however it introduced a regression where passing ``observed=False`` and
+``dropna=False`` to ``groupby`` would result in only observed categories. It was found
+that the patch fixing the ``dropna=False`` bug is incompatible with ``observed=False``,
+and decided that the best resolution is to restore the correct ``observed=False``
+behavior at the cost of reintroducing the ``dropna=False`` bug.
+
+.. ipython:: python
+
+   df = pd.DataFrame(
+       {
+           "x": pd.Categorical([1, None], categories=[1, 2, 3]),
+           "y": [3, 4],
+       }
+   )
+   df
+
+*1.5.0 behavior*:
+
+.. code-block:: ipython
+
+   In [3]: # Correct behavior, NA values are not dropped
+           df.groupby("x", observed=True, dropna=False).sum()
+   Out[3]:
+        y
+   x
+   1    3
+   NaN  4
+
+
+   In [4]: # Incorrect behavior, only observed categories present
+           df.groupby("x", observed=False, dropna=False).sum()
+   Out[4]:
+        y
+   x
+   1    3
+   NaN  4
+
+
+*1.5.1 behavior*:
+
+.. ipython:: python
+
+   # Incorrect behavior, NA values are dropped
+   df.groupby("x", observed=True, dropna=False).sum()
+
+   # Correct behavior, unobserved categories present (NA values still dropped)
+   df.groupby("x", observed=False, dropna=False).sum()
+
 .. _whatsnew_151.regressions:
 
 Fixed regressions

pandas/core/groupby/grouper.py

@@ -658,7 +658,7 @@ def group_index(self) -> Index:
 
     @cache_readonly
     def _codes_and_uniques(self) -> tuple[npt.NDArray[np.signedinteger], ArrayLike]:
-        if self._dropna and self._passed_categorical:
+        if self._passed_categorical:
             # we make a CategoricalIndex out of the cat grouper
             # preserving the categories / ordered attributes;
             # doesn't (yet - GH#46909) handle dropna=False

pandas/tests/groupby/conftest.py

@@ -24,6 +24,11 @@ def dropna(request):
     return request.param
 
 
+@pytest.fixture(params=[True, False])
+def observed(request):
+    return request.param
+
+
 @pytest.fixture
 def mframe(multiindex_dataframe_random_data):
     return multiindex_dataframe_random_data

pandas/tests/groupby/test_categorical.py

@@ -1828,3 +1828,20 @@ def test_groupby_categorical_aggregate_functions():
     )
 
     tm.assert_series_equal(result, expected)
+
+
+def test_groupby_categorical_dropna(observed, dropna):
+    # GH#48645 - dropna should have no impact on the result when there are no NA values
+    cat = Categorical([1, 2], categories=[1, 2, 3])
+    df = DataFrame({"x": Categorical([1, 2], categories=[1, 2, 3]), "y": [3, 4]})
+    gb = df.groupby("x", observed=observed, dropna=dropna)
+    result = gb.sum()
+
+    if observed:
+        expected = DataFrame({"y": [3, 4]}, index=cat)
+    else:
+        index = CategoricalIndex([1, 2, 3], [1, 2, 3])
+        expected = DataFrame({"y": [3, 4, 0]}, index=index)
+    expected.index.name = "x"
+
+    tm.assert_frame_equal(result, expected)

pandas/tests/groupby/test_groupby_dropna.py

@@ -408,7 +408,13 @@ def test_groupby_drop_nan_with_multi_index():
         ([2, np.nan, 1, 2], "Float32"),
         ([2, np.nan, 1, 2], "Int64"),
         ([2, np.nan, 1, 2], "Float64"),
-        (["y", None, "x", "y"], "category"),
+        pytest.param(
+            ["y", None, "x", "y"],
+            "category",
+            marks=pytest.mark.xfail(
+                reason="dropna=False not correct for categorical, GH#48645"
+            ),
+        ),
         (["y", pd.NA, "x", "y"], "string"),
         pytest.param(
             ["y", pd.NA, "x", "y"],