pythongh-119180: Fix annotations lookup on classes with custom metaclasses

See https://discuss.python.org/t/pep-749-implementing-pep-649/54974/28 and subsequent posts.

Author: JelleZijlstra

Include/internal/pycore_global_objects_fini_generated.h

@@ -82,6 +82,7 @@ struct _Py_global_strings {
         STRUCT_FOR_ID(__anext__)
         STRUCT_FOR_ID(__annotate__)
         STRUCT_FOR_ID(__annotations__)
+        STRUCT_FOR_ID(__annotations_cache__)
         STRUCT_FOR_ID(__args__)
         STRUCT_FOR_ID(__asyncio_running_event_loop__)
         STRUCT_FOR_ID(__await__)

Include/internal/pycore_global_strings.h

@@ -1,3 +1,4 @@
+import itertools
 import textwrap
 import types
 import unittest
@@ -16,22 +17,22 @@ def test_lazy_create_annotations(self):
         # a freshly created type shouldn't have an annotations dict yet.
         foo = type("Foo", (), {})
         for i in range(3):
-            self.assertFalse("__annotations__" in foo.__dict__)
+            self.assertFalse("__annotations_cache__" in foo.__dict__)
             d = foo.__annotations__
-            self.assertTrue("__annotations__" in foo.__dict__)
+            self.assertTrue("__annotations_cache__" in foo.__dict__)
             self.assertEqual(foo.__annotations__, d)
-            self.assertEqual(foo.__dict__['__annotations__'], d)
+            self.assertEqual(foo.__dict__['__annotations_cache__'], d)
             del foo.__annotations__
 
     def test_setting_annotations(self):
         foo = type("Foo", (), {})
         for i in range(3):
-            self.assertFalse("__annotations__" in foo.__dict__)
+            self.assertFalse("__annotations_cache__" in foo.__dict__)
             d = {'a': int}
             foo.__annotations__ = d
-            self.assertTrue("__annotations__" in foo.__dict__)
+            self.assertTrue("__annotations_cache__" in foo.__dict__)
             self.assertEqual(foo.__annotations__, d)
-            self.assertEqual(foo.__dict__['__annotations__'], d)
+            self.assertEqual(foo.__dict__['__annotations_cache__'], d)
             del foo.__annotations__
 
     def test_annotations_getset_raises(self):
@@ -55,9 +56,9 @@ class C:
             a:int=3
             b:str=4
         self.assertEqual(C.__annotations__, {"a": int, "b": str})
-        self.assertTrue("__annotations__" in C.__dict__)
+        self.assertTrue("__annotations_cache__" in C.__dict__)
         del C.__annotations__
-        self.assertFalse("__annotations__" in C.__dict__)
+        self.assertFalse("__annotations_cache__" in C.__dict__)
 
     def test_descriptor_still_works(self):
         class C:
@@ -270,6 +271,68 @@ def check_annotations(self, f):
         self.assertIs(f.__annotate__, None)
 
 
+class MetaclassTests(unittest.TestCase):
+    def test_annotated_meta(self):
+        class Meta(type):
+            a: int
+
+        class X(metaclass=Meta):
+            pass
+
+        class Y(metaclass=Meta):
+            b: float
+
+        self.assertEqual(Meta.__annotations__, {"a": int})
+        self.assertEqual(Meta.__annotate__(1), {"a": int})
+
+        self.assertEqual(X.__annotations__, {})
+        self.assertIs(X.__annotate__, None)
+
+        self.assertEqual(Y.__annotations__, {"b": float})
+        self.assertEqual(Y.__annotate__(1), {"b": float})
+
+    def test_ordering(self):
+        # Based on a sample by David Ellis
+        # https://discuss.python.org/t/pep-749-implementing-pep-649/54974/38
+
+        def make_classes():
+            class Meta(type):
+                a: int
+                expected_annotations = {"a": int}
+
+            class A(type, metaclass=Meta):
+                b: float
+                expected_annotations = {"b": float}
+
+            class B(metaclass=A):
+                c: str
+                expected_annotations = {"c": str}
+
+            class C(B):
+                expected_annotations = {}
+
+            class D(metaclass=Meta):
+                expected_annotations = {}
+
+            return Meta, A, B, C, D
+
+        classes = make_classes()
+        class_count = len(classes)
+        for order in itertools.permutations(range(class_count), class_count):
+            names = ", ".join(classes[i].__name__ for i in order)
+            with self.subTest(names=names):
+                classes = make_classes()  # Regenerate classes
+                for i in order:
+                    classes[i].__annotations__
+                for c in classes:
+                    with self.subTest(c=c):
+                        self.assertEqual(c.__annotations__, c.expected_annotations)
+                        if c.expected_annotations:
+                            self.assertEqual(c.__annotate__(1), c.expected_annotations)
+                        else:
+                            self.assertIs(c.__annotate__, None)
+
+
 class DeferredEvaluationTests(unittest.TestCase):
     def test_function(self):
         def func(x: undefined, /, y: undefined, *args: undefined, z: undefined, **kwargs: undefined) -> undefined:

Include/internal/pycore_runtime_init_generated.h

@@ -0,0 +1,2 @@
+Make lookup of ``__annotate__`` and ``__annotations__`` on classes more
+robust in the presence of metaclasses.

Include/internal/pycore_unicodeobject_generated.h

@@ -1902,7 +1902,7 @@ type_set_annotate(PyTypeObject *type, PyObject *value, void *Py_UNUSED(ignored))
         return -1;
     }
     if (!Py_IsNone(value)) {
-        if (PyDict_Pop(dict, &_Py_ID(__annotations__), NULL) == -1) {
+        if (PyDict_Pop(dict, &_Py_ID(__annotations_cache__), NULL) == -1) {
             Py_DECREF(dict);
             PyType_Modified(type);
             return -1;
@@ -1923,7 +1923,7 @@ type_get_annotations(PyTypeObject *type, void *context)
 
     PyObject *annotations;
     PyObject *dict = PyType_GetDict(type);
-    if (PyDict_GetItemRef(dict, &_Py_ID(__annotations__), &annotations) < 0) {
+    if (PyDict_GetItemRef(dict, &_Py_ID(__annotations_cache__), &annotations) < 0) {
         Py_DECREF(dict);
         return NULL;
     }
@@ -1962,7 +1962,7 @@ type_get_annotations(PyTypeObject *type, void *context)
         Py_DECREF(annotate);
         if (annotations) {
             int result = PyDict_SetItem(
-                    dict, &_Py_ID(__annotations__), annotations);
+                    dict, &_Py_ID(__annotations_cache__), annotations);
             if (result) {
                 Py_CLEAR(annotations);
             } else {
@@ -1988,10 +1988,10 @@ type_set_annotations(PyTypeObject *type, PyObject *value, void *context)
     PyObject *dict = PyType_GetDict(type);
     if (value != NULL) {
         /* set */
-        result = PyDict_SetItem(dict, &_Py_ID(__annotations__), value);
+        result = PyDict_SetItem(dict, &_Py_ID(__annotations_cache__), value);
     } else {
         /* delete */
-        result = PyDict_Pop(dict, &_Py_ID(__annotations__), NULL);
+        result = PyDict_Pop(dict, &_Py_ID(__annotations_cache__), NULL);
         if (result == 0) {
             PyErr_SetString(PyExc_AttributeError, "__annotations__");
             Py_DECREF(dict);
@@ -4223,6 +4223,24 @@ type_new_set_classcell(PyTypeObject *type)
     return 0;
 }
 
+static int
+type_new_set_annotate(PyTypeObject *type)
+{
+    PyObject *dict = lookup_tp_dict(type);
+    // If __annotate__ is not set (i.e., the class has no annotations),
+    // set it to None
+    int result = PyDict_Contains(dict, &_Py_ID(__annotate__));
+    if (result < 0) {
+        return -1;
+    }
+    else if (result == 0) {
+        if (PyDict_SetItem(dict, &_Py_ID(__annotate__), Py_None) < 0) {
+            return -1;
+        }
+    }
+    return 0;
+}
+
 static int
 type_new_set_classdictcell(PyTypeObject *type)
 {
@@ -4296,6 +4314,9 @@ type_new_set_attrs(const type_new_ctx *ctx, PyTypeObject *type)
     if (type_new_set_classdictcell(type) < 0) {
         return -1;
     }
+    if (type_new_set_annotate(type) < 0) {
+        return -1;
+    }
     return 0;
 }