fixup! BUG: Fix .groupby(categorical, sort=False) failing

Author: kernc

doc/source/whatsnew/v0.20.0.txt

@@ -539,7 +539,7 @@ Bug Fixes
 
 - Bug in ``resample``, where a non-string ```loffset`` argument would not be applied when resampling a timeseries (:issue:`13218`)
 
-- Bug in ``Categorical.unique()`` stripping unused categories resulting in ```.groupby(categorical, sort=False)`` not working (:issue:`13179`)
+- Bug in ``.groupby`` where ```.groupby(categorical, sort=False)`` would raise ``ValueError`` due to non-matching categories (:issue:`13179`)
 
 
 

pandas/core/categorical.py

@@ -1855,14 +1855,11 @@ def unique(self):
         cat = self.copy()
         # keep nan in codes
         cat._codes = unique_codes
-        if not self.ordered:
-            take_codes = unique_codes[unique_codes != -1]
-            # Sort categories according to codes order, suffixed by
-            # unused categories in original order (GH-13179)
-            order = (cat.categories[take_codes].tolist() +
-                     cat.categories.tolist()).index
-            cat.set_categories(sorted(cat.categories, key=order), inplace=True)
-        return cat
+        # exclude nan from indexer for categories
+        take_codes = unique_codes[unique_codes != -1]
+        if self.ordered:
+            take_codes = sorted(take_codes)
+        return cat.set_categories(cat.categories.take(take_codes))
 
     def equals(self, other):
         """

pandas/core/groupby.py

@@ -2315,6 +2315,9 @@ def __init__(self, index, grouper=None, obj=None, name=None, level=None,
                 # groupby
                 else:
                     cat = self.grouper.unique()
+                    cat.add_categories([c for c in self.grouper.categories
+                                        if c not in cat.categories],
+                                       inplace=True)  # GH-13179
                     self.grouper = self.grouper.reorder_categories(
                         cat.categories)
 

pandas/tests/test_categorical.py

@@ -1287,9 +1287,8 @@ def test_unique(self):
 
         cat = Categorical(["a", "b", "a", "a"], categories=["a", "b", "c"])
         res = cat.unique()
-        exp = Index(["a", "b", "c"])
         self.assert_index_equal(res.categories, exp)
-        tm.assert_categorical_equal(res, Categorical(["a", "b"], exp))
+        tm.assert_categorical_equal(res, Categorical(exp))
 
         cat = Categorical(["c", "a", "b", "a", "a"],
                           categories=["a", "b", "c"])
@@ -1303,9 +1302,9 @@ def test_unique(self):
         cat = Categorical(["b", np.nan, "b", np.nan, "a"],
                           categories=["a", "b", "c"])
         res = cat.unique()
-        exp = Index(["b", "a", "c"])
+        exp = Index(["b", "a"])
         self.assert_index_equal(res.categories, exp)
-        exp_cat = Categorical(["b", np.nan, "a"], categories=["b", "a", "c"])
+        exp_cat = Categorical(["b", np.nan, "a"], categories=["b", "a"])
         tm.assert_categorical_equal(res, exp_cat)
 
     def test_unique_ordered(self):
@@ -1325,14 +1324,13 @@ def test_unique_ordered(self):
         cat = Categorical(['b', 'a', 'a'], categories=['a', 'b', 'c'],
                           ordered=True)
         res = cat.unique()
-        exp_cat = Categorical(['b', 'a'], categories=['a', 'b', 'c'],
-                              ordered=True)
+        exp_cat = Categorical(['b', 'a'], categories=['a', 'b'], ordered=True)
         tm.assert_categorical_equal(res, exp_cat)
 
         cat = Categorical(['b', 'b', np.nan, 'a'], categories=['a', 'b', 'c'],
                           ordered=True)
         res = cat.unique()
-        exp_cat = Categorical(['b', np.nan, 'a'], categories=['a', 'b', 'c'],
+        exp_cat = Categorical(['b', np.nan, 'a'], categories=['a', 'b'],
                               ordered=True)
         tm.assert_categorical_equal(res, exp_cat)
 
@@ -1347,7 +1345,7 @@ def test_unique_index_series(self):
         tm.assert_categorical_equal(pd.Series(c).unique(), exp)
 
         c = Categorical([1, 1, 2, 2], categories=[3, 2, 1])
-        exp = Categorical([1, 2], categories=[1, 2, 3])
+        exp = Categorical([1, 2], categories=[1, 2])
         tm.assert_categorical_equal(c.unique(), exp)
         tm.assert_index_equal(Index(c).unique(), Index(exp))
         tm.assert_categorical_equal(pd.Series(c).unique(), exp)