Merge branch 'main' into web_benchmarks

Author: datapythonista

.github/actions/run-tests/action.yml

@@ -20,11 +20,6 @@ runs:
         path: test-data.xml
       if: failure()
 
-    - name: Report Coverage
-      run: coverage report -m
-      shell: bash -el {0}
-      if: failure()
-
     - name: Upload coverage to Codecov
       uses: codecov/codecov-action@v3
       with:

.github/workflows/code-checks.yml

@@ -86,7 +86,7 @@ jobs:
       if: ${{ steps.build.outcome == 'success' && always() }}
 
     - name: Typing + pylint
-      uses: pre-commit/[email protected].0
+      uses: pre-commit/[email protected].1
       with:
         extra_args: --verbose --hook-stage manual --all-files
       if: ${{ steps.build.outcome == 'success' && always() }}

.github/workflows/comment-commands.yml

@@ -24,12 +24,10 @@ jobs:
     concurrency:
       group: ${{ github.actor }}-preview-docs
     steps:
-      - run: |
-          if curl --output /dev/null --silent --head --fail "https://pandas.pydata.org/preview/${{ github.event.issue.number }}/"; then
-            curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d '{"body": "Website preview of this PR available at: https://pandas.pydata.org/preview/${{ github.event.issue.number }}/"}' https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.issue.number }}/comments
-          else
-            curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d '{"body": "No preview found for PR #${{ github.event.issue.number }}. Did the docs build complete?"}' https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.issue.number }}/comments
-          fi
+      - uses: pandas-dev/[email protected]
+        with:
+          previewer-server: "https://pandas.pydata.org/preview"
+          artifact-job: "Doc Build and Upload"
   asv_run:
     runs-on: ubuntu-22.04
     # TODO: Support more benchmarking options later, against different branches, against self, etc

.github/workflows/docbuild-and-upload.yml

@@ -90,10 +90,3 @@ jobs:
         name: website
         path: web/build
         retention-days: 14
-
-    - name: Trigger web/doc preview
-      run: curl -X POST https://pandas.pydata.org/preview/submit/$RUN_ID/$PR_ID/
-      env:
-        RUN_ID: ${{ github.run_id }}
-        PR_ID: ${{ github.event.pull_request.number }}
-      if: github.event_name == 'pull_request'

.github/workflows/package-checks.yml

@@ -24,7 +24,7 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        extra: ["test", "performance", "computation", "fss", "aws", "gcp", "excel", "parquet", "feather", "hdf5", "spss", "postgresql", "mysql", "sql-other", "html", "xml", "plot", "output-formatting", "clipboard", "compression", "consortium-standard", "all"]
+        extra: ["test", "pyarrow", "performance", "computation", "fss", "aws", "gcp", "excel", "parquet", "feather", "hdf5", "spss", "postgresql", "mysql", "sql-other", "html", "xml", "plot", "output-formatting", "clipboard", "compression", "all"]
       fail-fast: false
     name: Install Extras - ${{ matrix.extra }}
     concurrency:

.github/workflows/unit-tests.yml

@@ -57,45 +57,14 @@ jobs:
             # Also install zh_CN (its encoding is gb2312) but do not activate it.
             # It will be temporarily activated during tests with locale.setlocale
             extra_loc: "zh_CN"
-          - name: "Copy-on-Write 3.9"
-            env_file: actions-39.yaml
-            pattern: "not slow and not network and not single_cpu"
-            pandas_copy_on_write: "1"
-          - name: "Copy-on-Write 3.10"
-            env_file: actions-310.yaml
-            pattern: "not slow and not network and not single_cpu"
-            pandas_copy_on_write: "1"
-          - name: "Copy-on-Write 3.11"
-            env_file: actions-311.yaml
-            pattern: "not slow and not network and not single_cpu"
-            pandas_copy_on_write: "1"
-          - name: "Copy-on-Write 3.12"
-            env_file: actions-312.yaml
-            pattern: "not slow and not network and not single_cpu"
-            pandas_copy_on_write: "1"
-          - name: "Copy-on-Write 3.11 (warnings)"
-            env_file: actions-311.yaml
-            pattern: "not slow and not network and not single_cpu"
-            pandas_copy_on_write: "warn"
-          - name: "Copy-on-Write 3.10 (warnings)"
-            env_file: actions-310.yaml
-            pattern: "not slow and not network and not single_cpu"
-            pandas_copy_on_write: "warn"
-          - name: "Copy-on-Write 3.9 (warnings)"
-            env_file: actions-39.yaml
-            pattern: "not slow and not network and not single_cpu"
-            pandas_copy_on_write: "warn"
           - name: "Pypy"
             env_file: actions-pypy-39.yaml
             pattern: "not slow and not network and not single_cpu"
             test_args: "--max-worker-restart 0"
           - name: "Numpy Dev"
             env_file: actions-311-numpydev.yaml
             pattern: "not slow and not network and not single_cpu"
-            # Currently restricted the warnings that error to Deprecation Warnings from numpy
-            # done since pyarrow isn't compatible with numpydev always
-            # TODO: work with pyarrow to revert this?
-            test_args: "-W error::DeprecationWarning:numpy -W error::FutureWarning:numpy"
+            test_args: "-W error::DeprecationWarning -W error::FutureWarning"
           - name: "Pyarrow Nightly"
             env_file: actions-311-pyarrownightly.yaml
             pattern: "not slow and not network and not single_cpu"
@@ -113,7 +82,6 @@ jobs:
       PATTERN: ${{ matrix.pattern }}
       LANG: ${{ matrix.lang || 'C.UTF-8' }}
       LC_ALL: ${{ matrix.lc_all || '' }}
-      PANDAS_COPY_ON_WRITE: ${{ matrix.pandas_copy_on_write || '0' }}
       PANDAS_CI: ${{ matrix.pandas_ci || '1' }}
       TEST_ARGS: ${{ matrix.test_args || '' }}
       PYTEST_WORKERS: ${{ matrix.pytest_workers || 'auto' }}
@@ -123,7 +91,7 @@ jobs:
       QT_QPA_PLATFORM: offscreen
     concurrency:
       # https://github.community/t/concurrecy-not-work-for-push/183068/7
-      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.env_file }}-${{ matrix.pattern }}-${{ matrix.extra_apt || '' }}-${{ matrix.pandas_copy_on_write || '' }}
+      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.env_file }}-${{ matrix.pattern }}-${{ matrix.extra_apt || '' }}}
       cancel-in-progress: true
 
     services:
@@ -214,7 +182,8 @@ jobs:
     timeout-minutes: 90
     strategy:
       matrix:
-        os: [macos-latest, windows-latest]
+        # Note: Don't use macOS latest since macos 14 appears to be arm64 only
+        os: [macos-13, macos-14, windows-latest]
         env_file: [actions-39.yaml, actions-310.yaml, actions-311.yaml, actions-312.yaml]
       fail-fast: false
     runs-on: ${{ matrix.os }}
@@ -227,8 +196,7 @@ jobs:
       PANDAS_CI: 1
       PYTEST_TARGET: pandas
       PATTERN: "not slow and not db and not network and not single_cpu"
-      # GH 47443: PYTEST_WORKERS > 0 crashes Windows builds with memory related errors
-      PYTEST_WORKERS: ${{ matrix.os == 'macos-latest' && 'auto' || '0' }}
+      PYTEST_WORKERS: 'auto'
 
     steps:
       - name: Checkout
@@ -354,7 +322,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-22.04, macOS-latest, windows-latest]
+        # Separate out macOS 13 and 14, since macOS 14 is arm64 only
+        os: [ubuntu-22.04, macOS-13, macOS-14, windows-latest]
 
     timeout-minutes: 90
 

.github/workflows/wheels.yml

@@ -94,7 +94,9 @@ jobs:
         buildplat:
         - [ubuntu-22.04, manylinux_x86_64]
         - [ubuntu-22.04, musllinux_x86_64]
-        - [macos-12, macosx_*]
+        - [macos-12, macosx_x86_64]
+        # Note: M1 images on Github Actions start from macOS 14
+        - [macos-14, macosx_arm64]
         - [windows-2022, win_amd64]
         # TODO: support PyPy?
         python: [["cp39", "3.9"], ["cp310", "3.10"], ["cp311", "3.11"], ["cp312", "3.12"]]
@@ -128,7 +130,7 @@ jobs:
       # Python version used to build sdist doesn't matter
       # wheel will be built from sdist with the correct version
       - name: Unzip sdist (macOS)
-        if: ${{ matrix.buildplat[1] == 'macosx_*' }}
+        if: ${{ startsWith(matrix.buildplat[1], 'macosx') }}
         run: |
           tar -xzf ./dist/${{ env.sdist_name }} -C ./dist
 
@@ -139,18 +141,18 @@ jobs:
 
       - name: Build normal wheels
         if: ${{ (env.IS_SCHEDULE_DISPATCH != 'true' || env.IS_PUSH == 'true') }}
-        uses: pypa/[email protected].4
+        uses: pypa/[email protected].5
         with:
-         package-dir: ./dist/${{ matrix.buildplat[1] == 'macosx_*' && env.sdist_name || needs.build_sdist.outputs.sdist_file }}
+         package-dir: ./dist/${{ startsWith(matrix.buildplat[1], 'macosx') && env.sdist_name || needs.build_sdist.outputs.sdist_file }}
         env:
           CIBW_PRERELEASE_PYTHONS: True
           CIBW_BUILD: ${{ matrix.python[0] }}-${{ matrix.buildplat[1] }}
 
       - name: Build nightly wheels (with NumPy pre-release)
         if: ${{ (env.IS_SCHEDULE_DISPATCH == 'true' && env.IS_PUSH != 'true') }}
-        uses: pypa/[email protected].4
+        uses: pypa/[email protected].5
         with:
-         package-dir: ./dist/${{ matrix.buildplat[1] == 'macosx_*' && env.sdist_name || needs.build_sdist.outputs.sdist_file }}
+         package-dir: ./dist/${{ startsWith(matrix.buildplat[1], 'macosx') && env.sdist_name || needs.build_sdist.outputs.sdist_file }}
         env:
           # The nightly wheels should be build witht he NumPy 2.0 pre-releases
           # which requires the additional URL.
@@ -183,15 +185,15 @@ jobs:
           $TST_CMD = @"
           python -m pip install hypothesis>=6.46.1 pytest>=7.3.2 pytest-xdist>=2.2.0;
           python -m pip install `$(Get-Item pandas\wheelhouse\*.whl);
-          python -c `'import pandas as pd; pd.test(extra_args=[\"`\"--no-strict-data-files`\"\", \"`\"-m not clipboard and not single_cpu and not slow and not network and not db`\"\"])`';
+          python -c `'import pandas as pd; pd.test(extra_args=[`\"--no-strict-data-files`\", `\"-m not clipboard and not single_cpu and not slow and not network and not db`\"])`';
           "@
           # add rc to the end of the image name if the Python version is unreleased
           docker pull python:${{ matrix.python[1] == '3.12' && '3.12-rc' || format('{0}-windowsservercore', matrix.python[1]) }}
           docker run --env PANDAS_CI='1' -v ${PWD}:C:\pandas python:${{ matrix.python[1] == '3.12' && '3.12-rc' || format('{0}-windowsservercore', matrix.python[1]) }} powershell -Command $TST_CMD
 
       - uses: actions/upload-artifact@v4
         with:
-          name: ${{ matrix.python[0] }}-${{ startsWith(matrix.buildplat[1], 'macosx') && 'macosx' || matrix.buildplat[1] }}
+          name: ${{ matrix.python[0] }}-${{ matrix.buildplat[1] }}
           path: ./wheelhouse/*.whl
 
       - name: Upload wheels & sdist

.pre-commit-config.yaml

@@ -19,7 +19,7 @@ ci:
     skip: [pylint, pyright, mypy]
 repos:
 -   repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.1.6
+    rev: v0.1.13
     hooks:
     -   id: ruff
         args: [--exit-non-zero-on-fix]
@@ -31,8 +31,7 @@ repos:
         exclude: ^pandas/tests
         args: [--select, "ANN001,ANN2", --fix-only, --exit-non-zero-on-fix]
     -   id: ruff-format
-        # TODO: "." not needed in ruff 0.1.8
-        args: ["."]
+        exclude: ^scripts
 -   repo: https://github.com/jendrikseipp/vulture
     rev: 'v2.10'
     hooks:
@@ -54,13 +53,11 @@ repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.5.0
     hooks:
-    -   id: check-ast
     -   id: check-case-conflict
     -   id: check-toml
     -   id: check-xml
     -   id: check-yaml
         exclude: ^ci/meta.yaml$
-    -   id: debug-statements
     -   id: end-of-file-fixer
         exclude: \.txt$
     -   id: mixed-line-ending
@@ -101,8 +98,6 @@ repos:
 -   repo: https://github.com/pre-commit/pygrep-hooks
     rev: v1.10.0
     hooks:
-      - id: python-check-blanket-noqa
-      - id: python-check-blanket-type-ignore
       - id: rst-backticks
       - id: rst-directive-colons
         types: [text]  # overwrite types: [rst]
@@ -132,7 +127,7 @@ repos:
         types: [python]
         stages: [manual]
         additional_dependencies: &pyright_dependencies
-        - [email protected].347
+        - [email protected].351
     -   id: pyright
         # note: assumes python env is setup and activated
         name: pyright reportGeneralTypeIssues
@@ -211,39 +206,22 @@ repos:
         language: pygrep
         entry: |
             (?x)
-            # pytest.xfail instead of pytest.mark.xfail
-            pytest\.xfail
-
             # imports from pandas._testing instead of `import pandas._testing as tm`
-            |from\ pandas\._testing\ import
+            from\ pandas\._testing\ import
             |from\ pandas\ import\ _testing\ as\ tm
 
-            # No direct imports from conftest
-            |conftest\ import
-            |import\ conftest
-
             # pandas.testing instead of tm
             |pd\.testing\.
 
             # pd.api.types instead of from pandas.api.types import ...
             |(pd|pandas)\.api\.types\.
 
-            # np.testing, np.array_equal
-            |(numpy|np)(\.testing|\.array_equal)
-
-            # unittest.mock (use pytest builtin monkeypatch fixture instead)
-            |(unittest(\.| import )mock|mock\.Mock\(\)|mock\.patch)
+            # np.array_equal
+            |(numpy|np)\.array_equal
 
             # pytest raises without context
             |\s\ pytest.raises
 
-            # TODO
-            # pytest.warns (use tm.assert_produces_warning instead)
-            # |pytest\.warns
-
-            # os.remove
-            |os\.remove
-
             # Unseeded numpy default_rng
             |default_rng\(\)
         files: ^pandas/tests/
@@ -301,18 +279,6 @@ repos:
         files: ^pandas/core/
         exclude: ^pandas/core/api\.py$
         types: [python]
-    -   id: use-io-common-urlopen
-        name: Use pandas.io.common.urlopen instead of urllib.request.urlopen
-        language: python
-        entry: python scripts/use_io_common_urlopen.py
-        files: ^pandas/
-        exclude: ^pandas/tests/
-        types: [python]
-    -   id: no-bool-in-core-generic
-        name: Use bool_t instead of bool in pandas/core/generic.py
-        entry: python scripts/no_bool_in_generic.py
-        language: python
-        files: ^pandas/core/generic\.py$
     -   id: no-return-exception
         name: Use raise instead of return for exceptions
         language: pygrep

README.md

@@ -1,6 +1,7 @@
-<div align="center">
-  <img src="https://pandas.pydata.org/static/img/pandas.svg"><br>
-</div>
+<picture align="center">
+  <source media="(prefers-color-scheme: dark)" srcset="https://pandas.pydata.org/static/img/pandas_white.svg">
+  <img alt="Pandas Logo" src="https://pandas.pydata.org/static/img/pandas.svg">
+</picture>
 
 -----------------
 

asv_bench/asv.conf.json

@@ -41,7 +41,7 @@
     // pip (with all the conda available packages installed first,
     // followed by the pip installed packages).
     "matrix": {
-        "Cython": ["3.0.5"],
+        "Cython": ["3.0"],
         "matplotlib": [],
         "sqlalchemy": [],
         "scipy": [],

asv_bench/benchmarks/algos/isin.py

@@ -59,7 +59,8 @@ def setup(self, dtype):
         elif dtype in ["str", "string[python]", "string[pyarrow]"]:
             try:
                 self.series = Series(
-                    Index([f"i-{i}" for i in range(N)], dtype=object), dtype=dtype
+                    Index([f"i-{i}" for i in range(N)], dtype=object)._values,
+                    dtype=dtype,
                 )
             except ImportError as err:
                 raise NotImplementedError from err

asv_bench/benchmarks/frame_methods.py

@@ -159,12 +159,6 @@ def setup(self):
 
     def time_items(self):
         # (monitor no-copying behaviour)
-        if hasattr(self.df, "_item_cache"):
-            self.df._item_cache.clear()
-        for name, col in self.df.items():
-            pass
-
-    def time_items_cached(self):
         for name, col in self.df.items():
             pass
 
@@ -593,8 +587,6 @@ def setup(self):
         N = 10000
         # this is the worst case, where every column has NaNs.
         arr = np.random.randn(N, 100)
-        # NB: we need to set values in array, not in df.values, otherwise
-        #  the benchmark will be misleading for ArrayManager
         arr[::2] = np.nan
 
         self.df = DataFrame(arr)

asv_bench/benchmarks/index_object.py

@@ -136,10 +136,14 @@ def setup(self):
             self.int_idxs.append(i_idx)
             o_idx = i_idx.astype(str)
             self.object_idxs.append(o_idx)
+        self.same_range_idx = [self.range_idx] * N
 
     def time_append_range_list(self):
         self.range_idx.append(self.range_idxs)
 
+    def time_append_range_list_same(self):
+        self.range_idx.append(self.same_range_idx)
+
     def time_append_int_list(self):
         self.int_idx.append(self.int_idxs)