Merge branch 'nogil_dict_pydictvalues-updated' into nogil-integration

Author: colesbury

Include/cpython/pystats.h

@@ -80,7 +80,6 @@ typedef struct _object_stats {
     uint64_t dict_materialized_new_key;
     uint64_t dict_materialized_too_big;
     uint64_t dict_materialized_str_subclass;
-    uint64_t dict_dematerialized;
     uint64_t type_cache_hits;
     uint64_t type_cache_misses;
     uint64_t type_cache_dunder_hits;

Include/internal/pycore_code.h

@@ -79,7 +79,10 @@ typedef struct {
 typedef struct {
     uint16_t counter;
     uint16_t type_version[2];
-    uint16_t keys_version[2];
+    union {
+        uint16_t keys_version[2];
+        uint16_t dict_offset;
+    };
     uint16_t descr[4];
 } _PyLoadMethodCache;
 

Include/internal/pycore_dict.h

@@ -9,9 +9,10 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
-#include "pycore_freelist.h"      // _PyFreeListState
-#include "pycore_identifier.h"    // _Py_Identifier
-#include "pycore_object.h"        // PyDictOrValues
+#include "pycore_freelist.h"             // _PyFreeListState
+#include "pycore_identifier.h"           // _Py_Identifier
+#include "pycore_object.h"               // PyManagedDictPointer
+#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_LOAD_SSIZE_ACQUIRE
 
 // Unsafe flavor of PyDict_GetItemWithError(): no error checking
 extern PyObject* _PyDict_GetItemWithError(PyObject *dp, PyObject *key);
@@ -181,6 +182,10 @@ struct _dictkeysobject {
  * [-1] = prefix size. [-2] = used size. size[-2-n...] = insertion order.
  */
 struct _dictvalues {
+    uint8_t capacity;
+    uint8_t size;
+    uint8_t embedded;
+    uint8_t valid;
     PyObject *values[1];
 };
 
@@ -196,6 +201,7 @@ static inline void* _DK_ENTRIES(PyDictKeysObject *dk) {
     size_t index = (size_t)1 << dk->dk_log2_index_bytes;
     return (&indices[index]);
 }
+
 static inline PyDictKeyEntry* DK_ENTRIES(PyDictKeysObject *dk) {
     assert(dk->dk_kind == DICT_KEYS_GENERAL);
     return (PyDictKeyEntry*)_DK_ENTRIES(dk);
@@ -211,9 +217,6 @@ static inline PyDictUnicodeEntry* DK_UNICODE_ENTRIES(PyDictKeysObject *dk) {
 #define DICT_WATCHER_MASK ((1 << DICT_MAX_WATCHERS) - 1)
 #define DICT_WATCHER_AND_MODIFICATION_MASK ((1 << (DICT_MAX_WATCHERS + DICT_WATCHED_MUTATION_BITS)) - 1)
 
-#define DICT_VALUES_SIZE(values) ((uint8_t *)values)[-1]
-#define DICT_VALUES_USED_SIZE(values) ((uint8_t *)values)[-2]
-
 #ifdef Py_GIL_DISABLED
 #define DICT_NEXT_VERSION(INTERP) \
     (_Py_atomic_add_uint64(&(INTERP)->dict_state.global_version, DICT_VERSION_INCREMENT) + DICT_VERSION_INCREMENT)
@@ -246,25 +249,60 @@ _PyDict_NotifyEvent(PyInterpreterState *interp,
     return DICT_NEXT_VERSION(interp) | (mp->ma_version_tag & DICT_WATCHER_AND_MODIFICATION_MASK);
 }
 
-extern PyObject *_PyObject_MakeDictFromInstanceAttributes(PyObject *obj, PyDictValues *values);
-PyAPI_FUNC(bool) _PyObject_MakeInstanceAttributesFromDict(PyObject *obj, PyDictOrValues *dorv);
+extern PyObject *_PyObject_MaterializeManagedDict(PyObject *obj);
+
 PyAPI_FUNC(PyObject *)_PyDict_FromItems(
         PyObject *const *keys, Py_ssize_t keys_offset,
         PyObject *const *values, Py_ssize_t values_offset,
         Py_ssize_t length);
 
+static inline uint8_t *
+get_insertion_order_array(PyDictValues *values)
+{
+    return (uint8_t *)&values->values[values->capacity];
+}
+
 static inline void
 _PyDictValues_AddToInsertionOrder(PyDictValues *values, Py_ssize_t ix)
 {
     assert(ix < SHARED_KEYS_MAX_SIZE);
-    uint8_t *size_ptr = ((uint8_t *)values)-2;
-    int size = *size_ptr;
-    assert(size+2 < DICT_VALUES_SIZE(values));
-    size++;
-    size_ptr[-size] = (uint8_t)ix;
-    *size_ptr = size;
+    int size = values->size;
+    uint8_t *array = get_insertion_order_array(values);
+    assert(size < values->capacity);
+    assert(((uint8_t)ix) == ix);
+    array[size] = (uint8_t)ix;
+    values->size = size+1;
 }
 
+static inline size_t
+shared_keys_usable_size(PyDictKeysObject *keys)
+{
+    // dk_usable will decrease for each instance that is created and each
+    // value that is added.  dk_nentries will increase for each value that
+    // is added.  We want to always return the right value or larger.
+    // We therefore increase dk_nentries first and we decrease dk_usable
+    // second, and conversely here we read dk_usable first and dk_entries
+    // second (to avoid the case where we read entries before the increment
+    // and read usable after the decrement)
+    Py_ssize_t dk_usable = FT_ATOMIC_LOAD_SSIZE_ACQUIRE(keys->dk_usable);
+    Py_ssize_t dk_nentries = FT_ATOMIC_LOAD_SSIZE_ACQUIRE(keys->dk_nentries);
+    return dk_nentries + dk_usable;
+}
+
+static inline size_t
+_PyInlineValuesSize(PyTypeObject *tp)
+{
+    PyDictKeysObject *keys = ((PyHeapTypeObject*)tp)->ht_cached_keys;
+    assert(keys != NULL);
+    size_t size = shared_keys_usable_size(keys);
+    size_t prefix_size = _Py_SIZE_ROUND_UP(size, sizeof(PyObject *));
+    assert(prefix_size < 256);
+    return prefix_size + (size + 1) * sizeof(PyObject *);
+}
+
+int
+_PyDict_DetachFromObject(PyDictObject *dict, PyObject *obj);
+
 #ifdef __cplusplus
 }
 #endif

Include/internal/pycore_object.h

@@ -12,6 +12,7 @@ extern "C" {
 #include "pycore_gc.h"            // _PyObject_GC_IS_TRACKED()
 #include "pycore_emscripten_trampoline.h" // _PyCFunction_TrampolineCall()
 #include "pycore_interp.h"        // PyInterpreterState.gc
+#include "pycore_pyatomic_ft_wrappers.h"  // FT_ATOMIC_STORE_PTR_RELAXED
 #include "pycore_pystate.h"       // _PyInterpreterState_GET()
 
 /* Check if an object is consistent. For example, ensure that the reference
@@ -276,9 +277,8 @@ _PyObject_Init(PyObject *op, PyTypeObject *typeobj)
 {
     assert(op != NULL);
     Py_SET_TYPE(op, typeobj);
-    if (_PyType_HasFeature(typeobj, Py_TPFLAGS_HEAPTYPE)) {
-        Py_INCREF(typeobj);
-    }
+    assert(_PyType_HasFeature(typeobj, Py_TPFLAGS_HEAPTYPE) || _Py_IsImmortal(typeobj));
+    Py_INCREF(typeobj);
     _Py_NewReference(op);
 }
 
@@ -621,12 +621,11 @@ extern PyTypeObject* _PyType_CalculateMetaclass(PyTypeObject *, PyObject *);
 extern PyObject* _PyType_GetDocFromInternalDoc(const char *, const char *);
 extern PyObject* _PyType_GetTextSignatureFromInternalDoc(const char *, const char *, int);
 
-extern int _PyObject_InitializeDict(PyObject *obj);
-int _PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp);
-extern int _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values,
-                                          PyObject *name, PyObject *value);
-PyObject * _PyObject_GetInstanceAttribute(PyObject *obj, PyDictValues *values,
-                                        PyObject *name);
+void _PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp);
+extern int _PyObject_TryStoreInstanceAttribute(PyObject *obj,
+                                               PyObject *name, PyObject *value);
+extern int _PyObject_TryGetInstanceAttribute(PyObject *obj, PyObject *name,
+                                             PyObject **attr);
 
 #ifdef Py_GIL_DISABLED
 #  define MANAGED_DICT_OFFSET    (((Py_ssize_t)sizeof(PyObject *))*-1)
@@ -637,42 +636,36 @@ PyObject * _PyObject_GetInstanceAttribute(PyObject *obj, PyDictValues *values,
 #endif
 
 typedef union {
-    PyObject *dict;
-    /* Use a char* to generate a warning if directly assigning a PyDictValues */
-    char *values;
-} PyDictOrValues;
+    PyDictObject *dict;
+} PyManagedDictPointer;
 
-static inline PyDictOrValues *
-_PyObject_DictOrValuesPointer(PyObject *obj)
+static inline PyManagedDictPointer *
+_PyObject_ManagedDictPointer(PyObject *obj)
 {
     assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
-    return (PyDictOrValues *)((char *)obj + MANAGED_DICT_OFFSET);
-}
-
-static inline int
-_PyDictOrValues_IsValues(PyDictOrValues dorv)
-{
-    return ((uintptr_t)dorv.values) & 1;
+    return (PyManagedDictPointer *)((char *)obj + MANAGED_DICT_OFFSET);
 }
 
-static inline PyDictValues *
-_PyDictOrValues_GetValues(PyDictOrValues dorv)
+static inline PyDictObject *
+_PyObject_GetManagedDict(PyObject *obj)
 {
-    assert(_PyDictOrValues_IsValues(dorv));
-    return (PyDictValues *)(dorv.values + 1);
+    PyManagedDictPointer *dorv = _PyObject_ManagedDictPointer(obj);
+    return (PyDictObject *)FT_ATOMIC_LOAD_PTR_RELAXED(dorv->dict);
 }
 
-static inline PyObject *
-_PyDictOrValues_GetDict(PyDictOrValues dorv)
+static inline void
+_PyObject_SetManagedDict(PyObject *obj, PyObject *dict)
 {
-    assert(!_PyDictOrValues_IsValues(dorv));
-    return dorv.dict;
+    PyManagedDictPointer *ptr = _PyObject_ManagedDictPointer(obj);
+    FT_ATOMIC_STORE_PTR_RELEASE(ptr->dict, dict);
 }
 
-static inline void
-_PyDictOrValues_SetValues(PyDictOrValues *ptr, PyDictValues *values)
+static inline PyDictValues *
+_PyObject_InlineValues(PyObject *obj)
 {
-    ptr->values = ((char *)values) - 1;
+    assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
+    assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
+    return (PyDictValues *)((char *)obj + sizeof(PyObject));
 }
 
 extern PyObject ** _PyObject_ComputedDictPointer(PyObject *);

Include/internal/pycore_opcode_metadata.h

@@ -21,8 +21,14 @@ extern "C" {
 
 #ifdef Py_GIL_DISABLED
 #define FT_ATOMIC_LOAD_SSIZE(value) _Py_atomic_load_ssize(&value)
+#define FT_ATOMIC_LOAD_SSIZE_ACQUIRE(value) \
+    _Py_atomic_load_ssize_acquire(&value)
 #define FT_ATOMIC_LOAD_SSIZE_RELAXED(value) \
     _Py_atomic_load_ssize_relaxed(&value)
+#define FT_ATOMIC_LOAD_PTR_RELAXED(value) \
+    _Py_atomic_load_ptr_relaxed(&value)
+#define FT_ATOMIC_LOAD_UINT8_RELAXED(value) \
+    _Py_atomic_load_uint8_relaxed(&value)
 #define FT_ATOMIC_STORE_PTR_RELAXED(value, new_value) \
     _Py_atomic_store_ptr_relaxed(&value, new_value)
 #define FT_ATOMIC_STORE_PTR_RELEASE(value, new_value) \
@@ -31,7 +37,10 @@ extern "C" {
     _Py_atomic_store_ssize_relaxed(&value, new_value)
 #else
 #define FT_ATOMIC_LOAD_SSIZE(value) value
+#define FT_ATOMIC_LOAD_SSIZE_ACQUIRE(value) value
 #define FT_ATOMIC_LOAD_SSIZE_RELAXED(value) value
+#define FT_ATOMIC_LOAD_PTR_RELAXED(value) value
+#define FT_ATOMIC_LOAD_UINT8_RELAXED(value) value
 #define FT_ATOMIC_STORE_PTR_RELAXED(value, new_value) value = new_value
 #define FT_ATOMIC_STORE_PTR_RELEASE(value, new_value) value = new_value
 #define FT_ATOMIC_STORE_SSIZE_RELAXED(value, new_value) value = new_value

Include/internal/pycore_pyatomic_ft_wrappers.h

@@ -626,13 +626,18 @@ given type object has a specified feature.
 /* Track types initialized using _PyStaticType_InitBuiltin(). */
 #define _Py_TPFLAGS_STATIC_BUILTIN (1 << 1)
 
+/* The values array is placed inline directly after the rest of
+ * the object. Implies Py_TPFLAGS_HAVE_GC.
+ */
+#define Py_TPFLAGS_INLINE_VALUES (1 << 2)
+
 /* Placement of weakref pointers are managed by the VM, not by the type.
  * The VM will automatically set tp_weaklistoffset.
  */
 #define Py_TPFLAGS_MANAGED_WEAKREF (1 << 3)
 
 /* Placement of dict (and values) pointers are managed by the VM, not by the type.
- * The VM will automatically set tp_dictoffset.
+ * The VM will automatically set tp_dictoffset. Implies Py_TPFLAGS_HAVE_GC.
  */
 #define Py_TPFLAGS_MANAGED_DICT (1 << 4)
 

Include/internal/pycore_uop_ids.h

@@ -148,8 +148,8 @@ class C(): pass
             self.assertIn(b'MemoryError', out)
             *_, count = line.split(b' ')
             count = int(count)
-            self.assertLessEqual(count, i*5)
-            self.assertGreaterEqual(count, i*5-2)
+            self.assertLessEqual(count, i*10)
+            self.assertGreaterEqual(count, i*10-4)
 
 
 # Py_GIL_DISABLED requires mimalloc (not malloc)