Merge branch 'main' into superopt

* main:
  pythongh-103479: [Enum] require __new__ to be considered a data type (pythonGH-103495)
  pythongh-103365: [Enum] STRICT boundary corrections (pythonGH-103494)
  pythonGH-103488: Use return-offset, not yield-offset. (pythonGH-103502)
  pythongh-103088: Fix test_venv error message to avoid bytes/str warning (pythonGH-103500)
  pythonGH-103082: Turn on branch events for FOR_ITER instructions. (python#103507)
  pythongh-102978: Fix mock.patch function signatures for class and staticmethod decorators (python#103228)
  pythongh-103462: Ensure SelectorSocketTransport.writelines registers a writer when data is still pending (python#103463)
  pythongh-95299: Rework test_cppext.py to not invoke setup.py directly (python#103316)

Author: carljm

Doc/howto/enum.rst

@@ -865,17 +865,19 @@ Some rules:
 4. When another data type is mixed in, the :attr:`value` attribute is *not the
    same* as the enum member itself, although it is equivalent and will compare
    equal.
-5. %-style formatting:  ``%s`` and ``%r`` call the :class:`Enum` class's
+5. A ``data type`` is a mixin that defines :meth:`__new__`, or a
+   :class:`~dataclasses.dataclass`
+6. %-style formatting:  ``%s`` and ``%r`` call the :class:`Enum` class's
    :meth:`__str__` and :meth:`__repr__` respectively; other codes (such as
    ``%i`` or ``%h`` for IntEnum) treat the enum member as its mixed-in type.
-6. :ref:`Formatted string literals <f-strings>`, :meth:`str.format`,
+7. :ref:`Formatted string literals <f-strings>`, :meth:`str.format`,
    and :func:`format` will use the enum's :meth:`__str__` method.
 
 .. note::
 
    Because :class:`IntEnum`, :class:`IntFlag`, and :class:`StrEnum` are
    designed to be drop-in replacements for existing constants, their
-   :meth:`__str__` method has been reset to their data types
+   :meth:`__str__` method has been reset to their data types'
    :meth:`__str__` method.
 
 When to use :meth:`__new__` vs. :meth:`__init__`

Doc/library/enum.rst

@@ -696,7 +696,8 @@ Data Types
 
    .. attribute:: STRICT
 
-      Out-of-range values cause a :exc:`ValueError` to be raised::
+      Out-of-range values cause a :exc:`ValueError` to be raised. This is the
+      default for :class:`Flag`::
 
          >>> from enum import Flag, STRICT, auto
          >>> class StrictFlag(Flag, boundary=STRICT):
@@ -714,7 +715,7 @@ Data Types
    .. attribute:: CONFORM
 
       Out-of-range values have invalid values removed, leaving a valid *Flag*
-      value. This is the default for :class:`Flag`::
+      value::
 
          >>> from enum import Flag, CONFORM, auto
          >>> class ConformFlag(Flag, boundary=CONFORM):

Include/internal/pycore_frame.h

@@ -61,7 +61,13 @@ typedef struct _PyInterpreterFrame {
     // over, or (in the case of a newly-created frame) a totally invalid value:
     _Py_CODEUNIT *prev_instr;
     int stacktop;  /* Offset of TOS from localsplus  */
-    uint16_t yield_offset;
+    /* The return_offset determines where a `RETURN` should go in the caller,
+     * relative to `prev_instr`.
+     * It is only meaningful to the callee,
+     * so it needs to be set in any CALL (to a Python function)
+     * or SEND (to a coroutine or generator).
+     * If there is no callee, then it is meaningless. */
+    uint16_t return_offset;
     char owner;
     /* Locals and stack */
     PyObject *localsplus[1];
@@ -121,7 +127,7 @@ _PyFrame_Initialize(
     frame->stacktop = code->co_nlocalsplus;
     frame->frame_obj = NULL;
     frame->prev_instr = _PyCode_CODE(code) - 1;
-    frame->yield_offset = 0;
+    frame->return_offset = 0;
     frame->owner = FRAME_OWNED_BY_THREAD;
 
     for (int i = null_locals_from; i < code->co_nlocalsplus; i++) {

Lib/asyncio/selector_events.py

@@ -1176,6 +1176,9 @@ def writelines(self, list_of_data):
             return
         self._buffer.extend([memoryview(data) for data in list_of_data])
         self._write_ready()
+        # If the entire buffer couldn't be written, register a write handler
+        if self._buffer:
+            self._loop._add_writer(self._sock_fd, self._write_ready)
 
     def can_write_eof(self):
         return True

Lib/enum.py

@@ -275,6 +275,13 @@ def __set_name__(self, enum_class, member_name):
         enum_member.__objclass__ = enum_class
         enum_member.__init__(*args)
         enum_member._sort_order_ = len(enum_class._member_names_)
+
+        if Flag is not None and issubclass(enum_class, Flag):
+            enum_class._flag_mask_ |= value
+            if _is_single_bit(value):
+                enum_class._singles_mask_ |= value
+            enum_class._all_bits_ = 2 ** ((enum_class._flag_mask_).bit_length()) - 1
+
         # If another member with the same value was already defined, the
         # new member becomes an alias to the existing one.
         try:
@@ -532,12 +539,8 @@ def __new__(metacls, cls, bases, classdict, *, boundary=None, _simple=False, **k
         classdict['_use_args_'] = use_args
         #
         # convert future enum members into temporary _proto_members
-        # and record integer values in case this will be a Flag
-        flag_mask = 0
         for name in member_names:
             value = classdict[name]
-            if isinstance(value, int):
-                flag_mask |= value
             classdict[name] = _proto_member(value)
         #
         # house-keeping structures
@@ -554,8 +557,9 @@ def __new__(metacls, cls, bases, classdict, *, boundary=None, _simple=False, **k
                 boundary
                 or getattr(first_enum, '_boundary_', None)
                 )
-        classdict['_flag_mask_'] = flag_mask
-        classdict['_all_bits_'] = 2 ** ((flag_mask).bit_length()) - 1
+        classdict['_flag_mask_'] = 0
+        classdict['_singles_mask_'] = 0
+        classdict['_all_bits_'] = 0
         classdict['_inverted_'] = None
         try:
             exc = None
@@ -644,21 +648,10 @@ def __new__(metacls, cls, bases, classdict, *, boundary=None, _simple=False, **k
             ):
             delattr(enum_class, '_boundary_')
             delattr(enum_class, '_flag_mask_')
+            delattr(enum_class, '_singles_mask_')
             delattr(enum_class, '_all_bits_')
             delattr(enum_class, '_inverted_')
         elif Flag is not None and issubclass(enum_class, Flag):
-            # ensure _all_bits_ is correct and there are no missing flags
-            single_bit_total = 0
-            multi_bit_total = 0
-            for flag in enum_class._member_map_.values():
-                flag_value = flag._value_
-                if _is_single_bit(flag_value):
-                    single_bit_total |= flag_value
-                else:
-                    # multi-bit flags are considered aliases
-                    multi_bit_total |= flag_value
-            enum_class._flag_mask_ = single_bit_total
-            #
             # set correct __iter__
             member_list = [m._value_ for m in enum_class]
             if member_list != sorted(member_list):
@@ -974,6 +967,7 @@ def _find_data_repr_(mcls, class_name, bases):
 
     @classmethod
     def _find_data_type_(mcls, class_name, bases):
+        # a datatype has a __new__ method, or a __dataclass_fields__ attribute
         data_types = set()
         base_chain = set()
         for chain in bases:
@@ -986,7 +980,7 @@ def _find_data_type_(mcls, class_name, bases):
                     if base._member_type_ is not object:
                         data_types.add(base._member_type_)
                         break
-                elif '__new__' in base.__dict__ or '__init__' in base.__dict__:
+                elif '__new__' in base.__dict__ or '__dataclass_fields__' in base.__dict__:
                     data_types.add(candidate or base)
                     break
                 else:
@@ -1303,8 +1297,8 @@ def _reduce_ex_by_global_name(self, proto):
 class FlagBoundary(StrEnum):
     """
     control how out of range values are handled
-    "strict" -> error is raised
-    "conform" -> extra bits are discarded   [default for Flag]
+    "strict" -> error is raised             [default for Flag]
+    "conform" -> extra bits are discarded
     "eject" -> lose flag status
     "keep" -> keep flag status and all bits [default for IntFlag]
     """
@@ -1315,7 +1309,7 @@ class FlagBoundary(StrEnum):
 STRICT, CONFORM, EJECT, KEEP = FlagBoundary
 
 
-class Flag(Enum, boundary=CONFORM):
+class Flag(Enum, boundary=STRICT):
     """
     Support for flags
     """
@@ -1394,6 +1388,7 @@ def _missing_(cls, value):
         # - value must not include any skipped flags (e.g. if bit 2 is not
         #   defined, then 0d10 is invalid)
         flag_mask = cls._flag_mask_
+        singles_mask = cls._singles_mask_
         all_bits = cls._all_bits_
         neg_value = None
         if (
@@ -1425,7 +1420,8 @@ def _missing_(cls, value):
             value = all_bits + 1 + value
         # get members and unknown
         unknown = value & ~flag_mask
-        member_value = value & flag_mask
+        aliases = value & ~singles_mask
+        member_value = value & singles_mask
         if unknown and cls._boundary_ is not KEEP:
             raise ValueError(
                     '%s(%r) -->  unknown values %r [%s]'
@@ -1439,11 +1435,25 @@ def _missing_(cls, value):
             pseudo_member = cls._member_type_.__new__(cls, value)
         if not hasattr(pseudo_member, '_value_'):
             pseudo_member._value_ = value
-        if member_value:
-            pseudo_member._name_ = '|'.join([
-                m._name_ for m in cls._iter_member_(member_value)
-                ])
-            if unknown:
+        if member_value or aliases:
+            members = []
+            combined_value = 0
+            for m in cls._iter_member_(member_value):
+                members.append(m)
+                combined_value |= m._value_
+            if aliases:
+                value = member_value | aliases
+                for n, pm in cls._member_map_.items():
+                    if pm not in members and pm._value_ and pm._value_ & value == pm._value_:
+                        members.append(pm)
+                        combined_value |= pm._value_
+            unknown = value ^ combined_value
+            pseudo_member._name_ = '|'.join([m._name_ for m in members])
+            if not combined_value:
+                pseudo_member._name_ = None
+            elif unknown and cls._boundary_ is STRICT:
+                raise ValueError('%r: no members with value %r' % (cls, unknown))
+            elif unknown:
                 pseudo_member._name_ += '|%s' % cls._numeric_repr_(unknown)
         else:
             pseudo_member._name_ = None
@@ -1675,6 +1685,7 @@ def convert_class(cls):
             body['_boundary_'] = boundary or etype._boundary_
             body['_flag_mask_'] = None
             body['_all_bits_'] = None
+            body['_singles_mask_'] = None
             body['_inverted_'] = None
             body['__or__'] = Flag.__or__
             body['__xor__'] = Flag.__xor__
@@ -1750,7 +1761,8 @@ def convert_class(cls):
                     else:
                         multi_bits |= value
                     gnv_last_values.append(value)
-            enum_class._flag_mask_ = single_bits
+            enum_class._flag_mask_ = single_bits | multi_bits
+            enum_class._singles_mask_ = single_bits
             enum_class._all_bits_ = 2 ** ((single_bits|multi_bits).bit_length()) - 1
             # set correct __iter__
             member_list = [m._value_ for m in enum_class]

Lib/test/setup_testcppext.py

@@ -1,5 +1,6 @@
 # gh-91321: Build a basic C++ test extension to check that the Python C API is
 # compatible with C++ and does not emit C++ compiler warnings.
+import os
 import sys
 from test import support
 
@@ -25,14 +26,8 @@
 
 def main():
     cppflags = list(CPPFLAGS)
-    if '-std=c++03' in sys.argv:
-        sys.argv.remove('-std=c++03')
-        std = 'c++03'
-        name = '_testcpp03ext'
-    else:
-        # Python currently targets C++11
-        std = 'c++11'
-        name = '_testcpp11ext'
+    std = os.environ["CPYTHON_TEST_CPP_STD"]
+    name = os.environ["CPYTHON_TEST_EXT_NAME"]
 
     cppflags = [*CPPFLAGS, f'-std={std}']
 

Lib/test/setuptools-67.6.1-py3-none-any.whl

@@ -747,6 +747,48 @@ def test_write_sendmsg_no_data(self):
         self.assertFalse(self.sock.sendmsg.called)
         self.assertEqual(list_to_buffer([b'data']), transport._buffer)
 
+    @unittest.skipUnless(selector_events._HAS_SENDMSG, 'no sendmsg')
+    def test_writelines_sendmsg_full(self):
+        data = memoryview(b'data')
+        self.sock.sendmsg = mock.Mock()
+        self.sock.sendmsg.return_value = len(data)
+
+        transport = self.socket_transport(sendmsg=True)
+        transport.writelines([data])
+        self.assertTrue(self.sock.sendmsg.called)
+        self.assertFalse(self.loop.writers)
+
+    @unittest.skipUnless(selector_events._HAS_SENDMSG, 'no sendmsg')
+    def test_writelines_sendmsg_partial(self):
+        data = memoryview(b'data')
+        self.sock.sendmsg = mock.Mock()
+        self.sock.sendmsg.return_value = 2
+
+        transport = self.socket_transport(sendmsg=True)
+        transport.writelines([data])
+        self.assertTrue(self.sock.sendmsg.called)
+        self.assertTrue(self.loop.writers)
+
+    def test_writelines_send_full(self):
+        data = memoryview(b'data')
+        self.sock.send.return_value = len(data)
+        self.sock.send.fileno.return_value = 7
+
+        transport = self.socket_transport()
+        transport.writelines([data])
+        self.assertTrue(self.sock.send.called)
+        self.assertFalse(self.loop.writers)
+
+    def test_writelines_send_partial(self):
+        data = memoryview(b'data')
+        self.sock.send.return_value = 2
+        self.sock.send.fileno.return_value = 7
+
+        transport = self.socket_transport()
+        transport.writelines([data])
+        self.assertTrue(self.sock.send.called)
+        self.assertTrue(self.loop.writers)
+
     @unittest.skipUnless(selector_events._HAS_SENDMSG, 'no sendmsg')
     def test_write_sendmsg_full(self):
         data = memoryview(b'data')

Lib/test/test_asyncio/test_selector_events.py

@@ -1,6 +1,7 @@
 # gh-91321: Build a basic C++ test extension to check that the Python C API is
 # compatible with C++ and does not emit C++ compiler warnings.
 import os.path
+import shutil
 import sys
 import unittest
 import subprocess
@@ -39,6 +40,10 @@ def check_build(self, std_cpp03, extension_name):
             self._check_build(std_cpp03, extension_name)
 
     def _check_build(self, std_cpp03, extension_name):
+        pkg_dir = 'pkg'
+        os.mkdir(pkg_dir)
+        shutil.copy(SETUP_TESTCPPEXT, os.path.join(pkg_dir, "setup.py"))
+
         venv_dir = 'env'
         verbose = support.verbose
 
@@ -59,11 +64,15 @@ def _check_build(self, std_cpp03, extension_name):
             python = os.path.join(venv_dir, 'bin', python_exe)
 
         def run_cmd(operation, cmd):
+            env = os.environ.copy()
+            env['CPYTHON_TEST_CPP_STD'] = 'c++03' if std_cpp03 else 'c++11'
+            env['CPYTHON_TEST_EXT_NAME'] = extension_name
             if verbose:
                 print('Run:', ' '.join(cmd))
-                subprocess.run(cmd, check=True)
+                subprocess.run(cmd, check=True, env=env)
             else:
                 proc = subprocess.run(cmd,
+                                      env=env,
                                       stdout=subprocess.PIPE,
                                       stderr=subprocess.STDOUT,
                                       text=True)
@@ -72,16 +81,16 @@ def run_cmd(operation, cmd):
                     self.fail(
                         f"{operation} failed with exit code {proc.returncode}")
 
-        # Build the C++ extension
         cmd = [python, '-X', 'dev',
-               SETUP_TESTCPPEXT, 'build_ext', '--verbose']
-        if std_cpp03:
-            cmd.append('-std=c++03')
-        run_cmd('Build', cmd)
+               '-m', 'pip', 'install',
+               support.findfile('setuptools-67.6.1-py3-none-any.whl'),
+               support.findfile('wheel-0.40.0-py3-none-any.whl')]
+        run_cmd('Install build dependencies', cmd)
 
-        # Install the C++ extension
+        # Build and install the C++ extension
         cmd = [python, '-X', 'dev',
-               SETUP_TESTCPPEXT, 'install']
+               '-m', 'pip', 'install', '--no-build-isolation',
+               os.path.abspath(pkg_dir)]
         run_cmd('Install', cmd)
 
         # Do a reference run. Until we test that running python