Merge remote-tracking branch 'upstream/main' into main

Author: tpackard1

doc/source/development/maintaining.rst

@@ -458,8 +458,8 @@ which will be triggered when the tag is pushed.
     git checkout master
     git pull --ff-only upstream master
     git checkout -B RLS-<version>
-    sed -i 's/BUILD_COMMIT: "v.*/BUILD_COMMIT: "'<version>'"/' azure/windows.yml azure/posix.yml
-    sed -i 's/BUILD_COMMIT="v.*/BUILD_COMMIT="'<version>'"/' .travis.yml
+    sed -i 's/BUILD_COMMIT: "v.*/BUILD_COMMIT: "'v<version>'"/' azure/windows.yml azure/posix.yml
+    sed -i 's/BUILD_COMMIT="v.*/BUILD_COMMIT="'v<version>'"/' .travis.yml
     git commit -am "RLS <version>"
     git push -u origin RLS-<version>
 
@@ -474,14 +474,21 @@ which will be triggered when the tag is pushed.
 Post-Release
 ````````````
 
-1. Close the milestone and the issue for the released version.
+1. Update symlink to stable documentation by logging in to our web server, and
+   editing ``/var/www/html/pandas-docs/stable`` to point to ``version/<latest-version>``.
 
-2. Create a new issue for the next release, with the estimated date or release.
+2. If releasing a major or minor release, open a PR in our source code to update
+   ``web/pandas/versions.json``, to have the desired versions in the documentation
+   dropdown menu.
 
-3. Open a PR with the placeholder for the release notes of the next version. See
+3. Close the milestone and the issue for the released version.
+
+4. Create a new issue for the next release, with the estimated date of release.
+
+5. Open a PR with the placeholder for the release notes of the next version. See
    for example [the PR for 1.5.3](https://github.com/pandas-dev/pandas/pull/49843/files).
 
-4. Announce the new release in the official channels (use previous announcements
+6. Announce the new release in the official channels (use previous announcements
    for reference):
 
     - The pandas-dev and pydata mailing lists

doc/source/whatsnew/index.rst

@@ -24,6 +24,7 @@ Version 1.5
 .. toctree::
    :maxdepth: 2
 
+   v1.5.4
    v1.5.3
    v1.5.2
    v1.5.1

doc/source/whatsnew/v1.5.2.rst

@@ -43,4 +43,4 @@ Other
 Contributors
 ~~~~~~~~~~~~
 
-.. contributors:: v1.5.1..v1.5.2|HEAD
+.. contributors:: v1.5.1..v1.5.2

doc/source/whatsnew/v1.5.3.rst

@@ -55,4 +55,4 @@ Other
 Contributors
 ~~~~~~~~~~~~
 
-.. contributors:: v1.5.2..v1.5.3|HEAD
+.. contributors:: v1.5.2..v1.5.3

doc/source/whatsnew/v1.5.4.rst

@@ -0,0 +1,38 @@
+.. _whatsnew_154:
+
+What's new in 1.5.4 (March XX, 2023)
+--------------------------------------
+
+These are the changes in pandas 1.5.4. See :ref:`release` for a full changelog
+including other versions of pandas.
+
+{{ header }}
+
+.. ---------------------------------------------------------------------------
+.. _whatsnew_154.regressions:
+
+Fixed regressions
+~~~~~~~~~~~~~~~~~
+-
+
+.. ---------------------------------------------------------------------------
+.. _whatsnew_154.bug_fixes:
+
+Bug fixes
+~~~~~~~~~
+-
+
+.. ---------------------------------------------------------------------------
+.. _whatsnew_154.other:
+
+Other
+~~~~~
+-
+
+.. ---------------------------------------------------------------------------
+.. _whatsnew_154.contributors:
+
+Contributors
+~~~~~~~~~~~~
+
+.. contributors:: v1.5.3..v1.5.4|HEAD

doc/source/whatsnew/v2.0.0.rst

@@ -161,6 +161,7 @@ Other enhancements
 - Added :meth:`Index.infer_objects` analogous to :meth:`Series.infer_objects` (:issue:`50034`)
 - Added ``copy`` parameter to :meth:`Series.infer_objects` and :meth:`DataFrame.infer_objects`, passing ``False`` will avoid making copies for series or columns that are already non-object or where no better dtype can be inferred (:issue:`50096`)
 - :meth:`DataFrame.plot.hist` now recognizes ``xlabel`` and ``ylabel`` arguments (:issue:`49793`)
+- :meth:`Series.drop_duplicates` has gained ``ignore_index`` keyword to reset index (:issue:`48304`)
 - Improved error message in :func:`to_datetime` for non-ISO8601 formats, informing users about the position of the first error (:issue:`50361`)
 - Improved error message when trying to align :class:`DataFrame` objects (for example, in :func:`DataFrame.compare`) to clarify that "identically labelled" refers to both index and columns (:issue:`50083`)
 - Added :meth:`DatetimeIndex.as_unit` and :meth:`TimedeltaIndex.as_unit` to convert to different resolutions; supported resolutions are "s", "ms", "us", and "ns" (:issue:`50616`)
@@ -453,6 +454,37 @@ Now, the axes return an empty :class:`RangeIndex`.
    pd.Series().index
    pd.DataFrame().axes
 
+.. _whatsnew_200.api_breaking.to_latex:
+
+DataFrame to LaTeX has a new render engine
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The existing :meth:`DataFrame.to_latex` has been restructured to utilise the
+extended implementation previously available under :meth:`.Styler.to_latex`.
+The arguments signature is similar, albeit ``col_space`` has been removed since
+it is ignored by LaTeX engines. This render engine also requires ``jinja2`` as a
+dependency which needs to be installed, since rendering is based upon jinja2 templates.
+
+The pandas options below are no longer used and will be removed in future releases.
+The alternative options giving similar functionality are indicated below:
+
+- ``display.latex.escape``: replaced with ``styler.format.escape``,
+- ``display.latex.longtable``: replaced with ``styler.latex.environment``,
+- ``display.latex.multicolumn``, ``display.latex.multicolumn_format`` and
+  ``display.latex.multirow``: replaced with ``styler.sparse.rows``,
+  ``styler.sparse.columns``, ``styler.latex.multirow_align`` and
+  ``styler.latex.multicol_align``,
+- ``display.latex.repr``: replaced with ``styler.render.repr``,
+- ``display.max_rows`` and ``display.max_columns``: replace with
+  ``styler.render.max_rows``, ``styler.render.max_columns`` and
+  ``styler.render.max_elements``.
+
+Note that the behaviour of ``_repr_latex_`` is also changed. Previously
+setting ``display.latex.repr`` would generate LaTeX only when using nbconvert for a
+JupyterNotebook, and not when the user is running the notebook. Now the
+``styler.render.repr`` option allows control of the specific output
+within JupyterNotebooks for operations (not just on nbconvert). See :issue:`39911`.
+
 .. _whatsnew_200.api_breaking.deps:
 
 Increased minimum versions for dependencies
@@ -618,6 +650,7 @@ Removal of prior version deprecations/changes
 - Removed deprecated :meth:`.Styler.set_na_rep` and :meth:`.Styler.set_precision` (:issue:`49397`)
 - Removed deprecated :meth:`.Styler.where` (:issue:`49397`)
 - Removed deprecated :meth:`.Styler.render` (:issue:`49397`)
+- Removed deprecated argument ``col_space`` in :meth:`DataFrame.to_latex` (:issue:`47970`)
 - Removed deprecated argument ``null_color`` in :meth:`.Styler.highlight_null` (:issue:`49397`)
 - Removed deprecated argument ``check_less_precise`` in :meth:`.testing.assert_frame_equal`, :meth:`.testing.assert_extension_array_equal`, :meth:`.testing.assert_series_equal`,  :meth:`.testing.assert_index_equal` (:issue:`30562`)
 - Removed deprecated ``null_counts`` argument in :meth:`DataFrame.info`. Use ``show_counts`` instead (:issue:`37999`)
@@ -792,6 +825,7 @@ Removal of prior version deprecations/changes
 - Changed behavior of comparison of ``NaT`` with a ``datetime.date`` object; these now raise on inequality comparisons (:issue:`39196`)
 - Enforced deprecation of silently dropping columns that raised a ``TypeError`` in :class:`Series.transform` and :class:`DataFrame.transform` when used with a list or dictionary (:issue:`43740`)
 - Changed behavior of :meth:`DataFrame.apply` with list-like so that any partial failure will raise an error (:issue:`43740`)
+- Changed behaviour of :meth:`DataFrame.to_latex` to now use the Styler implementation via :meth:`.Styler.to_latex` (:issue:`47970`)
 - Changed behavior of :meth:`Series.__setitem__` with an integer key and a :class:`Float64Index` when the key is not present in the index; previously we treated the key as positional (behaving like ``series.iloc[key] = val``), now we treat it is a label (behaving like ``series.loc[key] = val``), consistent with :meth:`Series.__getitem__`` behavior (:issue:`33469`)
 - Removed ``na_sentinel`` argument from :func:`factorize`, :meth:`.Index.factorize`, and :meth:`.ExtensionArray.factorize` (:issue:`47157`)
 - Changed behavior of :meth:`Series.diff` and :meth:`DataFrame.diff` with :class:`ExtensionDtype` dtypes whose arrays do not implement ``diff``, these now raise ``TypeError`` rather than casting to numpy (:issue:`31025`)
@@ -871,6 +905,7 @@ Performance improvements
 - Performance improvement in :class:`Period` constructor when constructing from a string or integer (:issue:`38312`)
 - Performance improvement in :func:`to_datetime` when using ``'%Y%m%d'`` format (:issue:`17410`)
 - Performance improvement in :func:`to_datetime` when format is given or can be inferred (:issue:`50465`)
+- Performance improvement in :meth:`Series.median` for nullable dtypes (:issue:`50838`)
 - Performance improvement in :func:`read_csv` when passing :func:`to_datetime` lambda-function to ``date_parser`` and inputs have mixed timezone offsetes (:issue:`35296`)
 - Performance improvement in :func:`isna` and :func:`isnull` (:issue:`50658`)
 - Performance improvement in :meth:`.SeriesGroupBy.value_counts` with categorical dtype (:issue:`46202`)
@@ -1023,6 +1058,7 @@ I/O
 - Bug in :meth:`DataFrame.to_string` ignoring float formatter for extension arrays (:issue:`39336`)
 - Fixed memory leak which stemmed from the initialization of the internal JSON module (:issue:`49222`)
 - Fixed issue where :func:`json_normalize` would incorrectly remove leading characters from column names that matched the ``sep`` argument (:issue:`49861`)
+- Bug in :func:`read_csv` unnecessarily overflowing for extension array dtype when containing ``NA`` (:issue:`32134`)
 - Bug in :meth:`DataFrame.to_dict` not converting ``NA`` to ``None`` (:issue:`50795`)
 - Bug in :meth:`DataFrame.to_json` where it would segfault when failing to encode a string (:issue:`50307`)
 - Bug in :func:`read_xml` where file-like objects failed when iterparse is used (:issue:`50641`)

pandas/_libs/tslib.pyx

@@ -220,6 +220,19 @@ def format_array_from_datetime(
     return result
 
 
+cdef int64_t _wrapped_cast_from_unit(object val, str unit) except? -1:
+    """
+    Call cast_from_unit and re-raise OverflowError as OutOfBoundsDatetime
+    """
+    # See also timedeltas._maybe_cast_from_unit
+    try:
+        return cast_from_unit(val, unit)
+    except OverflowError as err:
+        raise OutOfBoundsDatetime(
+            f"cannot convert input {val} with the unit '{unit}'"
+        ) from err
+
+
 def array_with_unit_to_datetime(
     ndarray[object] values,
     str unit,
@@ -261,13 +274,10 @@ def array_with_unit_to_datetime(
         bint is_raise = errors=="raise"
         ndarray[int64_t] iresult
         object tz = None
-        bint is_ym
         float fval
 
     assert is_ignore or is_coerce or is_raise
 
-    is_ym = unit in "YM"
-
     if unit == "ns":
         result, tz = array_to_datetime(
             values.astype(object, copy=False),
@@ -292,19 +302,7 @@ def array_with_unit_to_datetime(
                 if val != val or val == NPY_NAT:
                     iresult[i] = NPY_NAT
                 else:
-                    if is_ym and is_float_object(val) and not val.is_integer():
-                        # Analogous to GH#47266 for Timestamp
-                        raise ValueError(
-                            f"Conversion of non-round float with unit={unit} "
-                            "is ambiguous"
-                        )
-
-                    try:
-                        iresult[i] = cast_from_unit(val, unit)
-                    except OverflowError:
-                        raise OutOfBoundsDatetime(
-                            f"cannot convert input {val} with the unit '{unit}'"
-                        )
+                    iresult[i] = _wrapped_cast_from_unit(val, unit)
 
             elif isinstance(val, str):
                 if len(val) == 0 or val in nat_strings:
@@ -319,23 +317,7 @@ def array_with_unit_to_datetime(
                             f"non convertible value {val} with the unit '{unit}'"
                         )
 
-                    if is_ym and not fval.is_integer():
-                        # Analogous to GH#47266 for Timestamp
-                        raise ValueError(
-                            f"Conversion of non-round float with unit={unit} "
-                            "is ambiguous"
-                        )
-
-                    try:
-                        iresult[i] = cast_from_unit(fval, unit)
-                    except ValueError:
-                        raise ValueError(
-                            f"non convertible value {val} with the unit '{unit}'"
-                        )
-                    except OverflowError:
-                        raise OutOfBoundsDatetime(
-                            f"cannot convert input {val} with the unit '{unit}'"
-                        )
+                    iresult[i] = _wrapped_cast_from_unit(fval, unit)
 
             else:
                 # TODO: makes more sense as TypeError, but that would be an

pandas/_libs/tslibs/conversion.pyx

@@ -108,6 +108,15 @@ cdef int64_t cast_from_unit(object ts, str unit) except? -1:
     if ts is None:
         return m
 
+    if unit in ["Y", "M"] and  is_float_object(ts) and not ts.is_integer():
+        # GH#47267 it is clear that 2 "M" corresponds to 1970-02-01,
+        #  but not clear what 2.5 "M" corresponds to, so we will
+        #  disallow that case.
+        raise ValueError(
+            f"Conversion of non-round float with unit={unit} "
+            "is ambiguous"
+        )
+
     # cast the unit, multiply base/frace separately
     # to avoid precision issues from float -> int
     base = <int64_t>ts
@@ -287,13 +296,6 @@ cdef _TSObject convert_to_tsobject(object ts, tzinfo tz, str unit,
                     # GH#47266 Avoid cast_from_unit, which would give weird results
                     #  e.g. with "Y" and 150.0 we'd get 2120-01-01 09:00:00
                     return convert_to_tsobject(int(ts), tz, unit, False, False)
-                else:
-                    # GH#47267 it is clear that 2 "M" corresponds to 1970-02-01,
-                    #  but not clear what 2.5 "M" corresponds to, so we will
-                    #  disallow that case.
-                    raise ValueError(
-                        f"Conversion of non-round float with unit={unit} is ambiguous."
-                    )
 
             ts = cast_from_unit(ts, unit)
             obj.value = ts

pandas/core/arrays/masked.py

@@ -1081,12 +1081,7 @@ def _reduce(self, name: str, *, skipna: bool = True, **kwargs):
         data = self._data
         mask = self._mask
 
-        # coerce to a nan-aware float if needed
-        # (we explicitly use NaN within reductions)
-        if self._hasna:
-            data = self.to_numpy("float64", na_value=np.nan)
-
-        # median, skew, kurt, idxmin, idxmax
+        # median, skew, kurt, sem
         op = getattr(nanops, f"nan{name}")
         result = op(data, axis=0, skipna=skipna, mask=mask, **kwargs)
 

pandas/core/arrays/numeric.py

@@ -285,7 +285,7 @@ def _from_sequence_of_strings(
     ) -> T:
         from pandas.core.tools.numeric import to_numeric
 
-        scalars = to_numeric(strings, errors="raise")
+        scalars = to_numeric(strings, errors="raise", use_nullable_dtypes=True)
         return cls._from_sequence(scalars, dtype=dtype, copy=copy)
 
     _HANDLED_TYPES = (np.ndarray, numbers.Number)