diff --git a/doc/source/whatsnew/v1.0.2.rst b/doc/source/whatsnew/v1.0.2.rst
index c9031ac1ae9fe..57ed6adf667c8 100644
--- a/doc/source/whatsnew/v1.0.2.rst
+++ b/doc/source/whatsnew/v1.0.2.rst
@@ -19,6 +19,7 @@ Fixed regressions
 - Fixed regression in :meth:`Series.align` when ``other`` is a DataFrame and ``method`` is not None (:issue:`31785`)
 - Fixed regression in :meth:`pandas.core.groupby.RollingGroupby.apply` where the ``raw`` parameter was ignored (:issue:`31754`)
 - Fixed regression in :meth:`rolling(..).corr() <pandas.core.window.Rolling.corr>` when using a time offset (:issue:`31789`)
+- Fixed regression where :func:`read_pickle` raised a ``UnicodeDecodeError`` when reading a py27 pickle with :class:`MultiIndex` column (:issue:`31988`).
 - Fixed regression in :class:`DataFrame` arithmetic operations with mis-matched columns (:issue:`31623`)
 -
 
diff --git a/pandas/io/pickle.py b/pandas/io/pickle.py
index e51f24b551f31..4e731b8ecca11 100644
--- a/pandas/io/pickle.py
+++ b/pandas/io/pickle.py
@@ -171,21 +171,22 @@ def read_pickle(
 
     # 1) try standard library Pickle
     # 2) try pickle_compat (older pandas version) to handle subclass changes
-
-    excs_to_catch = (AttributeError, ImportError, ModuleNotFoundError)
+    # 3) try pickle_compat with latin-1 encoding upon a UnicodeDecodeError
 
     try:
-        with warnings.catch_warnings(record=True):
-            # We want to silence any warnings about, e.g. moved modules.
-            warnings.simplefilter("ignore", Warning)
-            return pickle.load(f)
-    except excs_to_catch:
-        # e.g.
-        #  "No module named 'pandas.core.sparse.series'"
-        #  "Can't get attribute '__nat_unpickle' on <module 'pandas._libs.tslib"
-        return pc.load(f, encoding=None)
+        excs_to_catch = (AttributeError, ImportError, ModuleNotFoundError)
+        try:
+            with warnings.catch_warnings(record=True):
+                # We want to silence any warnings about, e.g. moved modules.
+                warnings.simplefilter("ignore", Warning)
+                return pickle.load(f)
+        except excs_to_catch:
+            # e.g.
+            #  "No module named 'pandas.core.sparse.series'"
+            #  "Can't get attribute '__nat_unpickle' on <module 'pandas._libs.tslib"
+            return pc.load(f, encoding=None)
     except UnicodeDecodeError:
-        # e.g. can occur for files written in py27; see GH#28645
+        # e.g. can occur for files written in py27; see GH#28645 and GH#31988
         return pc.load(f, encoding="latin-1")
     finally:
         f.close()
diff --git a/pandas/tests/io/data/pickle/test_mi_py27.pkl b/pandas/tests/io/data/pickle/test_mi_py27.pkl
new file mode 100644
index 0000000000000..89021dd828108
Binary files /dev/null and b/pandas/tests/io/data/pickle/test_mi_py27.pkl differ
diff --git a/pandas/tests/io/test_pickle.py b/pandas/tests/io/test_pickle.py
index 78b630bb5ada1..584a545769c4c 100644
--- a/pandas/tests/io/test_pickle.py
+++ b/pandas/tests/io/test_pickle.py
@@ -382,14 +382,23 @@ def test_read(self, protocol, get_random_path):
             tm.assert_frame_equal(df, df2)
 
 
-def test_unicode_decode_error(datapath):
+@pytest.mark.parametrize(
+    ["pickle_file", "excols"],
+    [
+        ("test_py27.pkl", pd.Index(["a", "b", "c"])),
+        (
+            "test_mi_py27.pkl",
+            pd.MultiIndex.from_arrays([["a", "b", "c"], ["A", "B", "C"]]),
+        ),
+    ],
+)
+def test_unicode_decode_error(datapath, pickle_file, excols):
     # pickle file written with py27, should be readable without raising
-    #  UnicodeDecodeError, see GH#28645
-    path = datapath("io", "data", "pickle", "test_py27.pkl")
+    #  UnicodeDecodeError, see GH#28645 and GH#31988
+    path = datapath("io", "data", "pickle", pickle_file)
     df = pd.read_pickle(path)
 
     # just test the columns are correct since the values are random
-    excols = pd.Index(["a", "b", "c"])
     tm.assert_index_equal(df.columns, excols)