DEPR: mismatched null-likes in tm.assert_foo_equal (#58023)

Author: jbrockmendel

Diff for: doc/source/whatsnew/v3.0.0.rst

@@ -211,6 +211,7 @@ Removal of prior version deprecations/changes
 - All arguments in :meth:`Index.sort_values` are now keyword only (:issue:`56493`)
 - All arguments in :meth:`Series.to_dict` are now keyword only (:issue:`56493`)
 - Changed the default value of ``observed`` in :meth:`DataFrame.groupby` and :meth:`Series.groupby` to ``True`` (:issue:`51811`)
+- Enforce deprecation in :func:`testing.assert_series_equal` and :func:`testing.assert_frame_equal` with object dtype and mismatched null-like values, which are now considered not-equal (:issue:`18463`)
 - Enforced deprecation disallowing parsing datetimes with mixed time zones unless user passes ``utc=True`` to :func:`to_datetime` (:issue:`57275`)
 - Enforced deprecation in :meth:`Series.value_counts` and :meth:`Index.value_counts` with object dtype performing dtype inference on the ``.index`` of the result (:issue:`56161`)
 - Enforced deprecation of :meth:`.DataFrameGroupBy.get_group` and :meth:`.SeriesGroupBy.get_group` allowing the ``name`` argument to be a non-tuple when grouping by a list of length 1 (:issue:`54155`)

Diff for: pandas/_libs/testing.pyx

@@ -1,6 +1,5 @@
 import cmath
 import math
-import warnings
 
 import numpy as np
 
@@ -18,7 +17,6 @@ from pandas._libs.util cimport (
     is_real_number_object,
 )
 
-from pandas.util._exceptions import find_stack_level
 
 from pandas.core.dtypes.missing import array_equivalent
 
@@ -188,15 +186,7 @@ cpdef assert_almost_equal(a, b,
             return True
         elif checknull(b):
             # GH#18463
-            warnings.warn(
-                f"Mismatched null-like values {a} and {b} found. In a future "
-                "version, pandas equality-testing functions "
-                "(e.g. assert_frame_equal) will consider these not-matching "
-                "and raise.",
-                FutureWarning,
-                stacklevel=find_stack_level(),
-            )
-            return True
+            raise AssertionError(f"Mismatched null-like values {a} != {b}")
         raise AssertionError(f"{a} != {b}")
     elif checknull(b):
         raise AssertionError(f"{a} != {b}")

Diff for: pandas/tests/util/test_assert_almost_equal.py

@@ -311,7 +311,7 @@ def test_assert_almost_equal_inf(a, b):
 
 @pytest.mark.parametrize("left", objs)
 @pytest.mark.parametrize("right", objs)
-def test_mismatched_na_assert_almost_equal_deprecation(left, right):
+def test_mismatched_na_assert_almost_equal(left, right):
     left_arr = np.array([left], dtype=object)
     right_arr = np.array([right], dtype=object)
 
@@ -331,19 +331,19 @@ def test_mismatched_na_assert_almost_equal_deprecation(left, right):
         )
 
     else:
-        with tm.assert_produces_warning(FutureWarning, match=msg):
+        with pytest.raises(AssertionError, match=msg):
             _assert_almost_equal_both(left, right, check_dtype=False)
 
         # TODO: to get the same deprecation in assert_numpy_array_equal we need
         #  to change/deprecate the default for strict_nan to become True
         # TODO: to get the same deprecation in assert_index_equal we need to
         #  change/deprecate array_equivalent_object to be stricter, as
         #  assert_index_equal uses Index.equal which uses array_equivalent.
-        with tm.assert_produces_warning(FutureWarning, match=msg):
+        with pytest.raises(AssertionError, match="Series are different"):
             tm.assert_series_equal(
                 Series(left_arr, dtype=object), Series(right_arr, dtype=object)
             )
-        with tm.assert_produces_warning(FutureWarning, match=msg):
+        with pytest.raises(AssertionError, match="DataFrame.iloc.* are different"):
             tm.assert_frame_equal(
                 DataFrame(left_arr, dtype=object), DataFrame(right_arr, dtype=object)
             )