Merge branch 'main' into complex-realimag-asdouble

Author: skirpichev

.github/workflows/build.yml

@@ -40,6 +40,7 @@ jobs:
       run-docs: ${{ steps.docs-changes.outputs.run-docs || false }}
       run_tests: ${{ steps.check.outputs.run_tests }}
       run_hypothesis: ${{ steps.check.outputs.run_hypothesis }}
+      run_cifuzz: ${{ steps.check.outputs.run_cifuzz }}
       config_hash: ${{ steps.config_hash.outputs.hash }}
     steps:
       - uses: actions/checkout@v4
@@ -76,6 +77,21 @@ jobs:
             echo "Run hypothesis tests"
             echo "run_hypothesis=true" >> $GITHUB_OUTPUT
           fi
+
+          # oss-fuzz maintains a configuration for fuzzing the main branch of
+          # CPython, so CIFuzz should be run only for code that is likely to be
+          # merged into the main branch; compatibility with older branches may
+          # be broken.
+          FUZZ_RELEVANT_FILES='(\.c$|\.h$|\.cpp$|^configure$|^\.github/workflows/build\.yml$|^Modules/_xxtestfuzz)'
+          if [ "$GITHUB_BASE_REF" = "main" ] && [ "$(git diff --name-only origin/$GITHUB_BASE_REF.. | grep -qE $FUZZ_RELEVANT_FILES; echo $?)" -eq 0 ]; then
+            # The tests are pretty slow so they are executed only for PRs
+            # changing relevant files.
+            echo "Run CIFuzz tests"
+            echo "run_cifuzz=true" >> $GITHUB_OUTPUT
+          else
+            echo "Branch too old for CIFuzz tests; or no C files were changed"
+            echo "run_cifuzz=false" >> $GITHUB_OUTPUT
+          fi
       - name: Compute hash for config cache key
         id: config_hash
         run: |
@@ -140,9 +156,6 @@ jobs:
         run: make regen-configure
       - name: Build CPython
         run: |
-          # Deepfreeze will usually cause global objects to be added or removed,
-          # so we run it before regen-global-objects gets rum (in regen-all).
-          make regen-deepfreeze
           make -j4 regen-all
           make regen-stdlib-module-names
       - name: Check for changes
@@ -182,7 +195,7 @@ jobs:
     - name: Display build info
       run: .\python.bat -m test.pythoninfo
     - name: Tests
-      run: .\PCbuild\rt.bat -p Win32 -d -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0
+      run: .\PCbuild\rt.bat -p Win32 -d -q --fast-ci
 
   build_win_amd64:
     name: 'Windows (x64)'
@@ -201,7 +214,7 @@ jobs:
     - name: Display build info
       run: .\python.bat -m test.pythoninfo
     - name: Tests
-      run: .\PCbuild\rt.bat -p x64 -d -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0
+      run: .\PCbuild\rt.bat -p x64 -d -q --fast-ci
 
   build_win_arm64:
     name: 'Windows (arm64)'
@@ -252,7 +265,7 @@ jobs:
     - name: Display build info
       run: make pythoninfo
     - name: Tests
-      run: make buildbottest TESTOPTS="-j4 -uall,-cpu"
+      run: make test
 
   build_ubuntu:
     name: 'Ubuntu'
@@ -261,7 +274,7 @@ jobs:
     needs: check_source
     if: needs.check_source.outputs.run_tests == 'true'
     env:
-      OPENSSL_VER: 1.1.1v
+      OPENSSL_VER: 3.0.11
       PYTHONSTRICTEXTENSIONBUILD: 1
     steps:
     - uses: actions/checkout@v4
@@ -319,7 +332,7 @@ jobs:
       run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw
     - name: Tests
       working-directory: ${{ env.CPYTHON_BUILDDIR }}
-      run: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu"
+      run: xvfb-run make test
 
   build_ubuntu_ssltests:
     name: 'Ubuntu SSL tests with OpenSSL'
@@ -330,7 +343,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        openssl_ver: [1.1.1v, 3.0.10, 3.1.2]
+        openssl_ver: [1.1.1w, 3.0.11, 3.1.3]
     env:
       OPENSSL_VER: ${{ matrix.openssl_ver }}
       MULTISSL_DIR: ${{ github.workspace }}/multissl
@@ -382,7 +395,7 @@ jobs:
     needs: check_source
     if: needs.check_source.outputs.run_tests == 'true' && needs.check_source.outputs.run_hypothesis == 'true'
     env:
-      OPENSSL_VER: 1.1.1v
+      OPENSSL_VER: 3.0.11
       PYTHONSTRICTEXTENSIONBUILD: 1
     steps:
     - uses: actions/checkout@v4
@@ -491,7 +504,7 @@ jobs:
     needs: check_source
     if: needs.check_source.outputs.run_tests == 'true'
     env:
-      OPENSSL_VER: 1.1.1v
+      OPENSSL_VER: 3.0.11
       PYTHONSTRICTEXTENSIONBUILD: 1
       ASAN_OPTIONS: detect_leaks=0:allocator_may_return_null=1:handle_segv=0
     steps:
@@ -535,7 +548,47 @@ jobs:
     - name: Display build info
       run: make pythoninfo
     - name: Tests
-      run: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu"
+      run: xvfb-run make test
+
+  # CIFuzz job based on https://google.github.io/oss-fuzz/getting-started/continuous-integration/
+  cifuzz:
+    name: CIFuzz
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    needs: check_source
+    if: needs.check_source.outputs.run_cifuzz == 'true'
+    permissions:
+      security-events: write
+    strategy:
+      fail-fast: false
+      matrix:
+        sanitizer: [address, undefined, memory]
+    steps:
+      - name: Build fuzzers (${{ matrix.sanitizer }})
+        id: build
+        uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master
+        with:
+          oss-fuzz-project-name: cpython3
+          sanitizer: ${{ matrix.sanitizer }}
+      - name: Run fuzzers (${{ matrix.sanitizer }})
+        uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master
+        with:
+          fuzz-seconds: 600
+          oss-fuzz-project-name: cpython3
+          output-sarif: true
+          sanitizer: ${{ matrix.sanitizer }}
+      - name: Upload crash
+        uses: actions/upload-artifact@v3
+        if: failure() && steps.build.outcome == 'success'
+        with:
+          name: ${{ matrix.sanitizer }}-artifacts
+          path: ./out/artifacts
+      - name: Upload SARIF
+        if: always() && steps.build.outcome == 'success'
+        uses: github/codeql-action/upload-sarif@v2
+        with:
+          sarif_file: cifuzz-sarif/results.sarif
+          checkout_path: cifuzz-sarif
 
   all-required-green:  # This job does nothing and is only used for the branch protection
     name: All required checks pass
@@ -553,6 +606,7 @@ jobs:
     - build_ubuntu_ssltests
     - test_hypothesis
     - build_asan
+    - cifuzz
 
     runs-on: ubuntu-latest
 
@@ -565,6 +619,7 @@ jobs:
           build_ubuntu_ssltests,
           build_win32,
           build_win_arm64,
+          cifuzz,
           test_hypothesis,
         allowed-skips: >-
           ${{
@@ -588,6 +643,13 @@ jobs:
             '
             || ''
           }}
+          ${{
+            !fromJSON(needs.check_source.outputs.run_cifuzz)
+            && '
+            cifuzz,
+            '
+            || ''
+          }}
           ${{
             !fromJSON(needs.check_source.outputs.run_hypothesis)
             && '

.github/workflows/lint.yml

@@ -7,7 +7,7 @@ permissions:
 
 env:
   FORCE_COLOR: 1
-  RUFF_FORMAT: github
+  RUFF_OUTPUT_FORMAT: github
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}

.github/workflows/reusable-docs.yml

@@ -74,7 +74,7 @@ jobs:
     - name: 'Set up Python'
       uses: actions/setup-python@v4
       with:
-        python-version: '3.11'  # known to work with Sphinx 3.2
+        python-version: '3.11'  # known to work with Sphinx 4.2
         cache: 'pip'
         cache-dependency-path: 'Doc/requirements-oldest-sphinx.txt'
     - name: 'Install build dependencies'

.pre-commit-config.yaml

@@ -1,11 +1,15 @@
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.0.288
+    rev: v0.0.292
     hooks:
       - id: ruff
         name: Run Ruff on Lib/test/
         args: [--exit-non-zero-on-fix]
         files: ^Lib/test/
+      - id: ruff
+        name: Run Ruff on Argument Clinic
+        args: [--exit-non-zero-on-fix, --config=Tools/clinic/.ruff.toml]
+        files: ^Tools/clinic/|Lib/test/test_clinic.py
 
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.4.0
@@ -17,7 +21,16 @@ repos:
         types: [python]
         exclude: Lib/test/tokenizedata/coding20731.py
       - id: trailing-whitespace
-        types_or: [c, python, rst]
+        types_or: [c, inc, python, rst]
+
+  - repo: local
+    hooks:
+      - id: python-file-whitespace
+        name: "Check Python file whitespace"
+        entry: 'python Tools/patchcheck/reindent.py --nobackup --newline LF'
+        language: 'system'
+        types: [python]
+        exclude: '^(Lib/test/tokenizedata/|Tools/c-analyzer/cpython/_parser).*$'
 
   - repo: https://github.com/sphinx-contrib/sphinx-lint
     rev: v0.6.8
@@ -26,3 +39,8 @@ repos:
         args: [--enable=default-role]
         files: ^Doc/|^Misc/NEWS.d/next/
         types: [rst]
+
+  - repo: meta
+    hooks:
+      - id: check-hooks-apply
+      - id: check-useless-excludes

Doc/Makefile

@@ -7,7 +7,6 @@
 PYTHON       = python3
 VENVDIR      = ./venv
 SPHINXBUILD  = PATH=$(VENVDIR)/bin:$$PATH sphinx-build
-SPHINXLINT   = PATH=$(VENVDIR)/bin:$$PATH sphinx-lint
 BLURB        = PATH=$(VENVDIR)/bin:$$PATH blurb
 JOBS         = auto
 PAPER        =

Doc/c-api/init.rst

@@ -870,6 +870,19 @@ code, or when embedding the Python interpreter:
    When the current thread state is ``NULL``, this issues a fatal error (so that
    the caller needn't check for ``NULL``).
 
+   See also :c:func:`PyThreadState_GetUnchecked`.
+
+
+.. c:function:: PyThreadState* PyThreadState_GetUnchecked()
+
+   Similar to :c:func:`PyThreadState_Get`, but don't kill the process with a
+   fatal error if it is NULL. The caller is responsible to check if the result
+   is NULL.
+
+   .. versionadded:: 3.13
+      In Python 3.5 to 3.12, the function was private and known as
+      ``_PyThreadState_UncheckedGet()``.
+
 
 .. c:function:: PyThreadState* PyThreadState_Swap(PyThreadState *tstate)
 

Doc/c-api/init_config.rst

@@ -878,6 +878,19 @@ PyConfig
 
       .. versionadded:: 3.12
 
+   .. c:member:: int cpu_count
+
+      If the value of :c:member:`~PyConfig.cpu_count` is not ``-1`` then it will
+      override the return values of :func:`os.cpu_count`,
+      :func:`os.process_cpu_count`, and :func:`multiprocessing.cpu_count`.
+
+      Configured by the :samp:`-X cpu_count={n|default}` command line
+      flag or the :envvar:`PYTHON_CPU_COUNT` environment variable.
+
+      Default: ``-1``.
+
+      .. versionadded:: 3.13
+
    .. c:member:: int isolated
 
       If greater than ``0``, enable isolated mode:

Doc/c-api/object.rst

@@ -489,3 +489,21 @@ Object Protocol
    :c:macro:`Py_TPFLAGS_ITEMS_AT_END` set.
 
    .. versionadded:: 3.12
+
+.. c:function:: int PyObject_VisitManagedDict(PyObject *obj, visitproc visit, void *arg)
+
+   Visit the managed dictionary of *obj*.
+
+   This function must only be called in a traverse function of the type which
+   has the :c:macro:`Py_TPFLAGS_MANAGED_DICT` flag set.
+
+   .. versionadded:: 3.13
+
+.. c:function:: void PyObject_ClearManagedDict(PyObject *obj)
+
+   Clear the managed dictionary of *obj*.
+
+   This function must only be called in a traverse function of the type which
+   has the :c:macro:`Py_TPFLAGS_MANAGED_DICT` flag set.
+
+   .. versionadded:: 3.13

Doc/c-api/sys.rst

@@ -291,19 +291,24 @@ accessible to C code.  They all work with the current interpreter thread's
    Raise an auditing event with any active hooks. Return zero for success
    and non-zero with an exception set on failure.
 
+   The *event* string argument must not be *NULL*.
+
    If any hooks have been added, *format* and other arguments will be used
    to construct a tuple to pass. Apart from ``N``, the same format characters
    as used in :c:func:`Py_BuildValue` are available. If the built value is not
-   a tuple, it will be added into a single-element tuple. (The ``N`` format
-   option consumes a reference, but since there is no way to know whether
-   arguments to this function will be consumed, using it may cause reference
-   leaks.)
+   a tuple, it will be added into a single-element tuple.
+
+   The ``N`` format option must not be used. It consumes a reference, but since
+   there is no way to know whether arguments to this function will be consumed,
+   using it may cause reference leaks.
 
    Note that ``#`` format characters should always be treated as
    :c:type:`Py_ssize_t`, regardless of whether ``PY_SSIZE_T_CLEAN`` was defined.
 
    :func:`sys.audit` performs the same function from Python code.
 
+   See also :c:func:`PySys_AuditTuple`.
+
    .. versionadded:: 3.8
 
    .. versionchanged:: 3.8.2
@@ -312,6 +317,14 @@ accessible to C code.  They all work with the current interpreter thread's
       unavoidable deprecation warning was raised.
 
 
+.. c:function:: int PySys_AuditTuple(const char *event, PyObject *args)
+
+   Similar to :c:func:`PySys_Audit`, but pass arguments as a Python object.
+   *args* must be a :class:`tuple`. To pass no arguments, *args* can be *NULL*.
+
+   .. versionadded:: 3.13
+
+
 .. c:function:: int PySys_AddAuditHook(Py_AuditHookFunction hook, void *userData)
 
    Append the callable *hook* to the list of active auditing hooks.