Partial Revert "BUG/API: concat with empty DataFrames or all-NA columns (pandas-dev#43507)"

This reverts commit 084c543.

Author: jorisvandenbossche

pandas/core/internals/concat.py

@@ -31,20 +31,26 @@
     is_1d_only_ea_dtype,
     is_datetime64tz_dtype,
     is_dtype_equal,
+    is_scalar,
     needs_i8_conversion,
 )
 from pandas.core.dtypes.concat import (
     cast_to_common_type,
     concat_compat,
 )
 from pandas.core.dtypes.dtypes import ExtensionDtype
-from pandas.core.dtypes.missing import is_valid_na_for_dtype
+from pandas.core.dtypes.missing import (
+    is_valid_na_for_dtype,
+    isna,
+    isna_all,
+)
 
 import pandas.core.algorithms as algos
 from pandas.core.arrays import (
     DatetimeArray,
     ExtensionArray,
 )
+from pandas.core.arrays.sparse import SparseDtype
 from pandas.core.construction import ensure_wrapped_if_datetimelike
 from pandas.core.internals.array_manager import (
     ArrayManager,
@@ -419,7 +425,29 @@ def is_na(self) -> bool:
         blk = self.block
         if blk.dtype.kind == "V":
             return True
-        return False
+
+        if not blk._can_hold_na:
+            return False
+
+        values = blk.values
+        if values.size == 0:
+            return True
+        if isinstance(values.dtype, SparseDtype):
+            return False
+
+        if values.ndim == 1:
+            # TODO(EA2D): no need for special case with 2D EAs
+            val = values[0]
+            if not is_scalar(val) or not isna(val):
+                # ideally isna_all would do this short-circuiting
+                return False
+            return isna_all(values)
+        else:
+            val = values[0][0]
+            if not is_scalar(val) or not isna(val):
+                # ideally isna_all would do this short-circuiting
+                return False
+            return all(isna_all(row) for row in values)
 
     def get_reindexed_values(self, empty_dtype: DtypeObj, upcasted_na) -> ArrayLike:
         values: ArrayLike
@@ -567,6 +595,9 @@ def _dtype_to_na_value(dtype: DtypeObj, has_none_blocks: bool):
         # different from missing.na_value_for_dtype
         return None
     elif dtype.kind in ["i", "u"]:
+        if not has_none_blocks:
+            # different from missing.na_value_for_dtype
+            return None
         return np.nan
     elif dtype.kind == "O":
         return np.nan

pandas/tests/frame/methods/test_append.py

@@ -159,7 +159,7 @@ def test_append_empty_dataframe(self):
         expected = df1.copy()
         tm.assert_frame_equal(result, expected)
 
-    def test_append_dtypes(self):
+    def test_append_dtypes(self, using_array_manager):
 
         # GH 5754
         # row appends of different dtypes (so need to do by-item)
@@ -183,7 +183,10 @@ def test_append_dtypes(self):
         expected = DataFrame(
             {"bar": Series([Timestamp("20130101"), np.nan], dtype="M8[ns]")}
         )
-        expected = expected.astype(object)
+        if using_array_manager:
+            # TODO(ArrayManager) decide on exact casting rules in concat
+            # With ArrayManager, all-NaN float is not ignored
+            expected = expected.astype(object)
         tm.assert_frame_equal(result, expected)
 
         df1 = DataFrame({"bar": Timestamp("20130101")}, index=range(1))
@@ -192,7 +195,9 @@ def test_append_dtypes(self):
         expected = DataFrame(
             {"bar": Series([Timestamp("20130101"), np.nan], dtype="M8[ns]")}
         )
-        expected = expected.astype(object)
+        if using_array_manager:
+            # With ArrayManager, all-NaN float is not ignored
+            expected = expected.astype(object)
         tm.assert_frame_equal(result, expected)
 
         df1 = DataFrame({"bar": np.nan}, index=range(1))
@@ -201,7 +206,9 @@ def test_append_dtypes(self):
         expected = DataFrame(
             {"bar": Series([np.nan, Timestamp("20130101")], dtype="M8[ns]")}
         )
-        expected = expected.astype(object)
+        if using_array_manager:
+            # With ArrayManager, all-NaN float is not ignored
+            expected = expected.astype(object)
         tm.assert_frame_equal(result, expected)
 
         df1 = DataFrame({"bar": Timestamp("20130101")}, index=range(1))

pandas/tests/indexing/test_partial.py

@@ -370,6 +370,33 @@ def test_partial_setting_mixed_dtype(self):
         df.loc[2] = df.loc[1]
         tm.assert_frame_equal(df, expected)
 
+        # columns will align
+        df = DataFrame(columns=["A", "B"])
+        df.loc[0] = Series(1, index=range(4))
+        tm.assert_frame_equal(df, DataFrame(columns=["A", "B"], index=[0]))
+
+        # columns will align
+        # TODO: it isn't great that this behavior depends on consolidation
+        df = DataFrame(columns=["A", "B"])._consolidate()
+        df.loc[0] = Series(1, index=["B"])
+
+        exp = DataFrame([[np.nan, 1]], columns=["A", "B"], index=[0], dtype="float64")
+        tm.assert_frame_equal(df, exp)
+
+        # list-like must conform
+        df = DataFrame(columns=["A", "B"])
+
+        msg = "cannot set a row with mismatched columns"
+        with pytest.raises(ValueError, match=msg):
+            df.loc[0] = [1, 2, 3]
+
+        # TODO: #15657, these are left as object and not coerced
+        df = DataFrame(columns=["A", "B"])
+        df.loc[3] = [6, 7]
+
+        exp = DataFrame([[6, 7]], index=[3], columns=["A", "B"], dtype="object")
+        tm.assert_frame_equal(df, exp)
+
     def test_series_partial_set(self):
         # partial set with new index
         # Regression from GH4825

pandas/tests/reshape/merge/test_merge.py

@@ -682,7 +682,7 @@ def _constructor(self):
 
         assert isinstance(result, NotADataFrame)
 
-    def test_join_append_timedeltas(self):
+    def test_join_append_timedeltas(self, using_array_manager):
         # timedelta64 issues with join/merge
         # GH 5695
 
@@ -696,9 +696,11 @@ def test_join_append_timedeltas(self):
             {
                 "d": [datetime(2013, 11, 5, 5, 56), datetime(2013, 11, 5, 5, 56)],
                 "t": [timedelta(0, 22500), timedelta(0, 22500)],
-            },
-            dtype=object,
+            }
         )
+        if using_array_manager:
+            # TODO(ArrayManager) decide on exact casting rules in concat
+            expected = expected.astype(object)
         tm.assert_frame_equal(result, expected)
 
     def test_join_append_timedeltas2(self):