Merge branch 'main' into pythongh-123849

Author: erlend-aasland

.devcontainer/Dockerfile

@@ -1,7 +1,5 @@
 {
-    "build": {
-        "dockerfile": "Dockerfile"
-    },
+    "image": "ghcr.io/python/devcontainer:2024.09.25.11038928730",
     "onCreateCommand": [
         // Install common tooling.
         "dnf",

.devcontainer/devcontainer.json

@@ -27,8 +27,6 @@ Lib/test/cjkencodings/*                    noeol
 Lib/test/tokenizedata/coding20731.py       noeol
 Lib/test/decimaltestdata/*.decTest         noeol
 Lib/test/test_email/data/*.txt             noeol
-Lib/test/test_importlib/resources/data01/*           noeol
-Lib/test/test_importlib/resources/namespacedata01/*  noeol
 Lib/test/xmltestdata/*                     noeol
 
 # Shell scripts should have LF even on Windows because of Cygwin

.gitattributes

@@ -33,11 +33,11 @@ Objects/frameobject.c         @markshannon
 Objects/call.c                @markshannon
 Python/ceval*.c               @markshannon
 Python/ceval*.h               @markshannon
+Python/codegen.c              @markshannon @iritkatriel
 Python/compile.c              @markshannon @iritkatriel
 Python/assemble.c             @markshannon @iritkatriel
 Python/flowgraph.c            @markshannon @iritkatriel
 Python/instruction_sequence.c @iritkatriel
-Python/ast_opt.c              @isidentical
 Python/bytecodes.c            @markshannon
 Python/optimizer*.c           @markshannon
 Python/optimizer_analysis.c   @Fidget-Spinner
@@ -157,10 +157,12 @@ Include/internal/pycore_time.h  @pganssle @abalkin
 /Tools/cases_generator/        @markshannon
 
 # AST
-Python/ast.c                  @isidentical @JelleZijlstra
-Parser/asdl.py                @isidentical @JelleZijlstra
-Parser/asdl_c.py              @isidentical @JelleZijlstra
-Lib/ast.py                    @isidentical @JelleZijlstra
+Python/ast.c                  @isidentical @JelleZijlstra @eclips4
+Python/ast_opt.c              @isidentical @eclips4
+Parser/asdl.py                @isidentical @JelleZijlstra @eclips4
+Parser/asdl_c.py              @isidentical @JelleZijlstra @eclips4
+Lib/ast.py                    @isidentical @JelleZijlstra @eclips4
+Lib/test/test_ast/            @eclips4
 
 # Mock
 /Lib/unittest/mock.py         @cjw296
@@ -205,7 +207,6 @@ Doc/c-api/stable.rst          @encukou
 **/*bisect*                   @rhettinger
 **/*heapq*                    @rhettinger
 **/*functools*                @rhettinger
-**/*decimal*                  @rhettinger
 
 **/*dataclasses*              @ericvsmith
 

.github/CODEOWNERS

@@ -32,6 +32,7 @@ body:
         - "3.10"
         - "3.11"
         - "3.12"
+        - "3.13"
         - "CPython main branch"
     validations:
       required: true

.github/ISSUE_TEMPLATE/crash.yml

@@ -195,13 +195,14 @@ jobs:
 
   build_ubuntu_ssltests:
     name: 'Ubuntu SSL tests with OpenSSL'
-    runs-on: ubuntu-22.04
+    runs-on: ${{ matrix.os }}
     timeout-minutes: 60
     needs: check_source
     if: needs.check_source.outputs.run_tests == 'true'
     strategy:
       fail-fast: false
       matrix:
+        os: [ubuntu-22.04]
         openssl_ver: [3.0.15, 3.1.7, 3.2.3, 3.3.2]
     env:
       OPENSSL_VER: ${{ matrix.openssl_ver }}
@@ -231,7 +232,7 @@ jobs:
       uses: actions/cache@v4
       with:
         path: ./multissl/openssl/${{ env.OPENSSL_VER }}
-        key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }}
+        key: ${{ matrix.os }}-multissl-openssl-${{ env.OPENSSL_VER }}
     - name: Install OpenSSL
       if: steps.cache-openssl.outputs.cache-hit != 'true'
       run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux
@@ -410,7 +411,7 @@ jobs:
       uses: actions/cache@v4
       with:
         path: ./multissl/openssl/${{ env.OPENSSL_VER }}
-        key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }}
+        key: ${{ matrix.os }}-multissl-openssl-${{ env.OPENSSL_VER }}
     - name: Install OpenSSL
       if: steps.cache-openssl.outputs.cache-hit != 'true'
       run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux

.github/workflows/build.yml

@@ -110,17 +110,15 @@ jobs:
       - name: Native Windows
         if: runner.os == 'Windows' && matrix.architecture != 'ARM64'
         run: |
-          choco upgrade llvm -y
-          choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}
+          choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}.1.0
           ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '--pgo' }} -p ${{ matrix.architecture }}
           ./PCbuild/rt.bat ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} -q --multiprocess 0 --timeout 4500 --verbose2 --verbose3
 
       # No PGO or tests (yet):
       - name: Emulated Windows
         if: runner.os == 'Windows' && matrix.architecture == 'ARM64'
         run: |
-          choco upgrade llvm -y
-          choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}
+          choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}.1.0
           ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }}
 
       - name: Native macOS
@@ -159,7 +157,7 @@ jobs:
           CC="${{ matrix.compiler == 'clang' && 'clang --target=$HOST' || '$HOST-gcc' }}" \
             CPP="$CC --preprocess" \
             HOSTRUNNER=qemu-${{ matrix.architecture }} \
-            ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }} --build=x86_64-linux-gnu --host="$HOST" --with-build-python=../build/bin/python3 --with-pkg-config=no ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptc=no ac_cv_file__dev_ptmx=yes
+            ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--with-lto' }} --build=x86_64-linux-gnu --host="$HOST" --with-build-python=../build/bin/python3 --with-pkg-config=no ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptc=no ac_cv_file__dev_ptmx=yes
           make all --jobs 4
           ./python -m test --ignorefile=Tools/jit/ignore-tests-emulated-linux.txt --multiprocess 0 --timeout 4500 --verbose2 --verbose3
 

.github/workflows/jit.yml

@@ -35,7 +35,7 @@ jobs:
         path: config.cache
         key: ${{ github.job }}-${{ inputs.os }}-${{ env.IMAGE_VERSION }}-${{ inputs.config_hash }}
     - name: Install Homebrew dependencies
-      run: brew install pkg-config [email protected] xz gdbm tcl-tk
+      run: brew install pkg-config [email protected] xz gdbm tcl-tk make
     - name: Configure CPython
       run: |
         GDBM_CFLAGS="-I$(brew --prefix gdbm)/include" \
@@ -44,14 +44,27 @@ jobs:
           --config-cache \
           --with-pydebug \
           --enable-slower-safety \
+          --enable-safety \
           ${{ inputs.free-threading && '--disable-gil' || '' }} \
           --prefix=/opt/python-dev \
           --with-openssl="$(brew --prefix [email protected])"
     - name: Build CPython
-      run: set -o pipefail; make -j8 2>&1 | tee compiler_output.txt
+      if : ${{ inputs.free-threading || inputs.os != 'macos-13' }}
+      run: gmake -j8
+    - name: Build CPython for compiler warning check
+      if : ${{ !inputs.free-threading && inputs.os == 'macos-13' }}
+      run: set -o pipefail; gmake -j8 --output-sync 2>&1 | tee compiler_output_macos.txt
     - name: Display build info
       run: make pythoninfo
     - name: Check compiler warnings
-      run: python3 Tools/build/check_warnings.py --compiler-output-file-path=compiler_output.txt --warning-ignore-file-path=Tools/build/.warningignore_macos --compiler-output-type=clang
+      if : ${{ !inputs.free-threading && inputs.os == 'macos-13' }}
+      run: >-
+        python3 Tools/build/check_warnings.py
+        --compiler-output-file-path=compiler_output_macos.txt
+        --warning-ignore-file-path=Tools/build/.warningignore_macos
+        --compiler-output-type=clang
+        --fail-on-regression
+        --fail-on-improvement
+        --path-prefix="./"
     - name: Tests
       run: make test

.github/workflows/reusable-macos.yml

@@ -14,7 +14,11 @@ jobs:
   build_ubuntu_reusable:
     name: 'build and test'
     timeout-minutes: 60
-    runs-on: ubuntu-22.04
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-22.04]
     env:
       FORCE_COLOR: 1
       OPENSSL_VER: 3.0.15
@@ -36,7 +40,7 @@ jobs:
       uses: actions/cache@v4
       with:
         path: ./multissl/openssl/${{ env.OPENSSL_VER }}
-        key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }}
+        key: ${{ matrix.os }}-multissl-openssl-${{ env.OPENSSL_VER }}
     - name: Install OpenSSL
       if: steps.cache-openssl.outputs.cache-hit != 'true'
       run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux
@@ -67,20 +71,33 @@ jobs:
       working-directory: ${{ env.CPYTHON_BUILDDIR }}
       run: >-
         ../cpython-ro-srcdir/configure
-        CFLAGS="-fdiagnostics-format=json"
         --config-cache
         --with-pydebug
         --enable-slower-safety
+        --enable-safety
         --with-openssl=$OPENSSL_DIR
         ${{ fromJSON(inputs.free-threading) && '--disable-gil' || '' }}
     - name: Build CPython out-of-tree
+      if: ${{ inputs.free-threading }}
       working-directory: ${{ env.CPYTHON_BUILDDIR }}
-      run: set -o pipefail; make -j4 2>&1 | tee compiler_output.txt
+      run: make -j4
+    - name: Build CPython out-of-tree (for compiler warning check)
+      if: ${{ !inputs.free-threading}}
+      working-directory: ${{ env.CPYTHON_BUILDDIR }}
+      run: set -o pipefail; make -j4 --output-sync 2>&1 | tee compiler_output_ubuntu.txt
     - name: Display build info
       working-directory: ${{ env.CPYTHON_BUILDDIR }}
       run: make pythoninfo
     - name: Check compiler warnings
-      run: python Tools/build/check_warnings.py --compiler-output-file-path=${{ env.CPYTHON_BUILDDIR }}/compiler_output.txt --warning-ignore-file-path ${GITHUB_WORKSPACE}/Tools/build/.warningignore_ubuntu --compiler-output-type=json
+      if: ${{ !inputs.free-threading }}
+      run: >-
+        python Tools/build/check_warnings.py
+        --compiler-output-file-path=${{ env.CPYTHON_BUILDDIR }}/compiler_output_ubuntu.txt
+        --warning-ignore-file-path ${GITHUB_WORKSPACE}/Tools/build/.warningignore_ubuntu
+        --compiler-output-type=gcc
+        --fail-on-regression
+        --fail-on-improvement
+        --path-prefix="../cpython-ro-srcdir/"
     - name: Remount sources writable for tests
       # some tests write to srcdir, lack of pyc files slows down testing
       run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw

.github/workflows/reusable-ubuntu.yml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.3.4
+    rev: v0.6.7
     hooks:
       - id: ruff
         name: Run Ruff (lint) on Doc/
@@ -10,6 +10,10 @@ repos:
         name: Run Ruff (lint) on Lib/test/
         args: [--exit-non-zero-on-fix]
         files: ^Lib/test/
+      - id: ruff
+        name: Run Ruff (lint) on Tools/build/check_warnings.py
+        args: [--exit-non-zero-on-fix, --config=Tools/build/.ruff.toml]
+        files: ^Tools/build/check_warnings.py
       - id: ruff
         name: Run Ruff (lint) on Argument Clinic
         args: [--exit-non-zero-on-fix, --config=Tools/clinic/.ruff.toml]
@@ -20,15 +24,20 @@ repos:
         files: ^Doc/
 
   - repo: https://github.com/psf/black-pre-commit-mirror
-    rev: 24.4.2
+    rev: 24.8.0
     hooks:
+      - id: black
+        name: Run Black on Tools/build/check_warnings.py
+        files: ^Tools/build/check_warnings.py
+        language_version: python3.12
+        args: [--line-length=79]
       - id: black
         name: Run Black on Tools/jit/
         files: ^Tools/jit/
         language_version: python3.12
 
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.5.0
+    rev: v4.6.0
     hooks:
       - id: check-case-conflict
       - id: check-merge-conflict
@@ -42,7 +51,7 @@ repos:
         types_or: [c, inc, python, rst]
 
   - repo: https://github.com/sphinx-contrib/sphinx-lint
-    rev: v0.9.1
+    rev: v1.0.0
     hooks:
       - id: sphinx-lint
         args: [--enable=default-role]

.pre-commit-config.yaml

@@ -1,19 +1,19 @@
 # Python for Android
 
 These instructions are only needed if you're planning to compile Python for
-Android yourself. Most users should *not* need to do this. If you're looking to
-use Python on Android, one of the following tools will provide a much more
-approachable user experience:
-
-* [Briefcase](https://briefcase.readthedocs.io), from the BeeWare project
-* [Buildozer](https://buildozer.readthedocs.io), from the Kivy project
-* [Chaquopy](https://chaquo.com/chaquopy/)
+Android yourself. Most users should *not* need to do this. Instead, use one of
+the tools listed in `Doc/using/android.rst`, which will provide a much easier
+experience.
 
 
 ## Prerequisites
 
-Export the `ANDROID_HOME` environment variable to point at your Android SDK. If
-you don't already have the SDK, here's how to install it:
+First, make sure you have all the usual tools and libraries needed to build
+Python for your development machine.
+
+Second, you'll need an Android SDK. If you already have the SDK installed,
+export the `ANDROID_HOME` environment variable to point at its location.
+Otherwise, here's how to install it:
 
 * Download the "Command line tools" from <https://developer.android.com/studio>.
 * Create a directory `android-sdk/cmdline-tools`, and unzip the command line
@@ -37,11 +37,6 @@ development tools, which currently means Linux or macOS. This involves doing a
 cross-build where you use a "build" Python (for your development machine) to
 help produce a "host" Python for Android.
 
-First, make sure you have all the usual tools and libraries needed to build
-Python for your development machine. The only Android tool you need to install
-is the command line tools package above: the build script will download the
-rest.
-
 The easiest way to do a build is to use the `android.py` script. You can either
 have it perform the entire build process from start to finish in one step, or
 you can do it in discrete steps that mirror running `configure` and `make` for
@@ -80,17 +75,20 @@ call. For example, if you want a pydebug build that also caches the results from
 
 ## Testing
 
-The tests can be run on Linux, macOS, or Windows, although on Windows you'll
-have to build the `cross-build/HOST` subdirectory on one of the other platforms
-and copy it over.
+The test suite can be run on Linux, macOS, or Windows:
 
-The test suite can usually be run on a device with 2 GB of RAM, though for some
-configurations or test orders you may need to increase this. As of Android
+* On Linux, the emulator needs access to the KVM virtualization interface, and
+  a DISPLAY environment variable pointing at an X server.
+* On Windows, you won't be able to do the build on the same machine, so you'll
+  have to copy the `cross-build/HOST` directory from somewhere else.
+
+The test suite can usually be run on a device with 2 GB of RAM, but this is
+borderline, so you may need to increase it to 4 GB. As of Android
 Studio Koala, 2 GB is the default for all emulators, although the user interface
-may indicate otherwise. The effective setting is `hw.ramSize` in
-~/.android/avd/*.avd/hardware-qemu.ini, whereas Android Studio displays the
-value from config.ini. Changing the value in Android Studio will update both of
-these files.
+may indicate otherwise. Locate the emulator's directory under `~/.android/avd`,
+and find `hw.ramSize` in both config.ini and hardware-qemu.ini. Either set these
+manually to the same value, or use the Android Studio Device Manager, which will
+update both files.
 
 Before running the test suite, follow the instructions in the previous section
 to build the architecture you want to test. Then run the test script in one of
@@ -131,3 +129,8 @@ Every time you run `android.py test`, changes in pure-Python files in the
 repository's `Lib` directory will be picked up immediately. Changes in C files,
 and architecture-specific files such as sysconfigdata, will not take effect
 until you re-run `android.py make-host` or `build`.
+
+
+## Using in your own app
+
+See `Doc/using/android.rst`.