_PyObject_VAR_SIZE: always round up to a multiple-of-pointer-size value.

As Guido suggested, this makes the new subclassing code substantially
simpler.  But the mechanics of doing it w/ C macro semantics are a mess,
and _PyObject_VAR_SIZE has a new calling sequence now.

Question:  The PyObject_NEW_VAR macro appears to be part of the public API.
Regardless of what it expands to, the notion that it has to round up the
memory it allocates is new, and extensions containing the old
PyObject_NEW_VAR macro expansion (which was embedded in the
PyObject_NEW_VAR expansion) won't do this rounding.  But the rounding
isn't actually *needed* except for new-style instances with dict pointers
after a variable-length blob of embedded data.  So my guess is that we do
not need to bump the API version for this (as the rounding isn't needed
for anything an extension can do unless it's recompiled anyway).  What's
your guess?

Author: tim-one

Include/objimpl.h

@@ -56,14 +56,14 @@ form of memory management you're using).
 Unless you have specific memory management requirements, it is
 recommended to use PyObject_{New, NewVar, Del}. */
 
-/* 
+/*
  * Core object memory allocator
  * ============================
  */
 
 /* The purpose of the object allocator is to make the distinction
    between "object memory" and the rest within the Python heap.
-   
+
    Object memory is the one allocated by PyObject_{New, NewVar}, i.e.
    the one that holds the object's representation defined by its C
    type structure, *excluding* any object-specific memory buffers that
@@ -172,16 +172,41 @@ extern DL_IMPORT(void) _PyObject_Del(PyObject *);
 	( (op)->ob_size = (size), PyObject_INIT((op), (typeobj)) )
 
 #define _PyObject_SIZE(typeobj) ( (typeobj)->tp_basicsize )
-#define _PyObject_VAR_SIZE(typeobj, n) \
-	( (typeobj)->tp_basicsize + (n) * (typeobj)->tp_itemsize )
+
+/* _PyObject_VAR_SIZE computes the amount of memory allocated for a vrbl-
+  size object with nitems items, exclusive of gc overhead (if any).  The
+  value is rounded up to the closest multiple of sizeof(void *), in order
+  to ensure that pointer fields at the end of the object are correctly
+  aligned for the platform (this is of special importance for subclasses
+  of, e.g., str or long, so that pointers can be stored after the embedded
+  data).
+
+  Note that there's no memory wastage in doing this, as malloc has to
+  return (at worst) pointer-aligned memory anyway
+
+  However, writing the macro to *return* the result is clumsy due to the
+  calculations needed.  Instead you must pass the result lvalue as the first
+  argument, and it should be of type size_t (both because that's the
+  correct conceptual type, and because using an unsigned type allows the
+  compiler to generate faster code for the mod computation inside the
+  macro).
+*/
+#define _PyObject_VAR_SIZE(result, typeobj, nitems)			\
+	do {								\
+    		size_t mod;						\
+		(result) = (size_t) (typeobj)->tp_basicsize;		\
+		(result) += (size_t) ((nitems)*(typeobj)->tp_itemsize);	\
+		mod = (result) % SIZEOF_VOID_P;				\
+		if (mod)						\
+			(result) += SIZEOF_VOID_P - mod;		\
+    	} while(0)
 
 #define PyObject_NEW(type, typeobj) \
 ( (type *) PyObject_Init( \
 	(PyObject *) PyObject_MALLOC( _PyObject_SIZE(typeobj) ), (typeobj)) )
-#define PyObject_NEW_VAR(type, typeobj, n) \
-( (type *) PyObject_InitVar( \
-	(PyVarObject *) PyObject_MALLOC( _PyObject_VAR_SIZE((typeobj),(n)) ),\
-	(typeobj), (n)) )
+
+#define PyObject_NEW_VAR(type, typeobj, nitems) \
+	((type *) _PyObject_NewVar(typeobj, nitems))
 
 #define PyObject_DEL(op) PyObject_FREE(op)
 
@@ -230,8 +255,7 @@ extern DL_IMPORT(void) _PyObject_Del(PyObject *);
 #define PyObject_IS_GC(o) (PyType_IS_GC((o)->ob_type) && \
 	((o)->ob_type->tp_is_gc == NULL || (o)->ob_type->tp_is_gc(o)))
 
-extern DL_IMPORT(PyObject *) _PyObject_GC_Malloc(PyTypeObject *,
-					int nitems, size_t padding);
+extern DL_IMPORT(PyObject *) _PyObject_GC_Malloc(PyTypeObject *, int);
 extern DL_IMPORT(PyVarObject *) _PyObject_GC_Resize(PyVarObject *, int);
 
 #define PyObject_GC_Resize(type, op, n) \
@@ -276,7 +300,7 @@ extern PyGC_Head _PyGC_generation0;
 
 #define PyObject_GC_Track(op) _PyObject_GC_Track((PyObject *)op)
 #define PyObject_GC_UnTrack(op) _PyObject_GC_UnTrack((PyObject *)op)
-	
+
 
 #define PyObject_GC_New(type, typeobj) \
 		( (type *) _PyObject_GC_New(typeobj) )

Modules/gcmodule.c

@@ -798,14 +798,17 @@ _PyObject_GC_UnTrack(PyObject *op)
 }
 
 PyObject *
-_PyObject_GC_Malloc(PyTypeObject *tp, int nitems, size_t padding)
+_PyObject_GC_Malloc(PyTypeObject *tp, int nitems)
 {
 	PyObject *op;
+	size_t basicsize;
 #ifdef WITH_CYCLE_GC
-	const size_t basic = (size_t)_PyObject_VAR_SIZE(tp, nitems);
-	const size_t nbytes = sizeof(PyGC_Head) + basic + padding;
+	size_t nbytes;
+	PyGC_Head *g;
 
-	PyGC_Head *g = PyObject_MALLOC(nbytes);
+	_PyObject_VAR_SIZE(basicsize, tp, nitems);
+	nbytes = sizeof(PyGC_Head) + basicsize;
+	g = PyObject_MALLOC(nbytes);
 	if (g == NULL)
 		return (PyObject *)PyErr_NoMemory();
 	g->gc_next = NULL;
@@ -821,7 +824,8 @@ _PyObject_GC_Malloc(PyTypeObject *tp, int nitems, size_t padding)
 	}
 	op = FROM_GC(g);
 #else
-	op = PyObject_MALLOC(_PyObject_VAR_SIZE(tp, nitems) + padding);
+	_PyObject_VAR_SIZE(basicsize, tp, nitems);
+	op = PyObject_MALLOC(basicsize);
 	if (op == NULL)
 		return (PyObject *)PyErr_NoMemory();
 
@@ -832,33 +836,36 @@ _PyObject_GC_Malloc(PyTypeObject *tp, int nitems, size_t padding)
 PyObject *
 _PyObject_GC_New(PyTypeObject *tp)
 {
-	PyObject *op = _PyObject_GC_Malloc(tp, 0, 0);
+	PyObject *op = _PyObject_GC_Malloc(tp, 0);
 	return PyObject_INIT(op, tp);
 }
 
 PyVarObject *
-_PyObject_GC_NewVar(PyTypeObject *tp, int size)
+_PyObject_GC_NewVar(PyTypeObject *tp, int nitems)
 {
-	PyVarObject *op = (PyVarObject *) _PyObject_GC_Malloc(tp, size, 0);
-	return PyObject_INIT_VAR(op, tp, size);
+	PyVarObject *op = (PyVarObject *) _PyObject_GC_Malloc(tp, nitems);
+	return PyObject_INIT_VAR(op, tp, nitems);
 }
 
 PyVarObject *
-_PyObject_GC_Resize(PyVarObject *op, int size)
+_PyObject_GC_Resize(PyVarObject *op, int nitems)
 {
+	size_t basicsize;
 #ifdef WITH_CYCLE_GC
 	PyGC_Head *g = AS_GC(op);
-	g = PyObject_REALLOC(g, _PyObject_VAR_SIZE(op->ob_type, size) +
-						sizeof(PyGC_Head));
+
+	_PyObject_VAR_SIZE(basicsize, op->ob_type, nitems);
+	g = PyObject_REALLOC(g,  sizeof(PyGC_Head) + basicsize);
 	if (g == NULL)
 		return (PyVarObject *)PyErr_NoMemory();
 	op = (PyVarObject *) FROM_GC(g);
 #else
-	op = PyObject_REALLOC(op, _PyObject_VAR_SIZE(op->ob_type, size));
+	_PyObject_VAR_SIZE(basicsize, op->ob_type, nitems);
+	op = PyObject_REALLOC(op, basicsize);
 	if (op == NULL)
 		return (PyVarObject *)PyErr_NoMemory();
 #endif
-	op->ob_size = size;
+	op->ob_size = nitems;
 	return op;
 }
 

Objects/object.c

@@ -127,13 +127,16 @@ _PyObject_New(PyTypeObject *tp)
 }
 
 PyVarObject *
-_PyObject_NewVar(PyTypeObject *tp, int size)
+_PyObject_NewVar(PyTypeObject *tp, int nitems)
 {
 	PyVarObject *op;
-	op = (PyVarObject *) PyObject_MALLOC(_PyObject_VAR_SIZE(tp, size));
+	size_t size;
+
+	_PyObject_VAR_SIZE(size, tp, nitems);
+	op = (PyVarObject *) PyObject_MALLOC(size);
 	if (op == NULL)
 		return (PyVarObject *)PyErr_NoMemory();
-	return PyObject_INIT_VAR(op, tp, size);
+	return PyObject_INIT_VAR(op, tp, nitems);
 }
 
 void
@@ -1146,8 +1149,6 @@ PyObject_SetAttr(PyObject *v, PyObject *name, PyObject *value)
 PyObject **
 _PyObject_GetDictPtr(PyObject *obj)
 {
-#define PTRSIZE (sizeof(PyObject *))
-
 	long dictoffset;
 	PyTypeObject *tp = obj->ob_type;
 
@@ -1157,19 +1158,11 @@ _PyObject_GetDictPtr(PyObject *obj)
 	if (dictoffset == 0)
 		return NULL;
 	if (dictoffset < 0) {
-		/* dictoffset is positive by the time we're ready to round
-		   it, and compilers can generate faster rounding code if
-		   they know that. */
-		unsigned long udo;  /* unsigned dictoffset */
-		const long nitems = ((PyVarObject *)obj)->ob_size;
-		const long size = _PyObject_VAR_SIZE(tp, nitems);
-
-		dictoffset += size;
-		assert(dictoffset > 0); /* Sanity check */
-		/* Round up to multiple of PTRSIZE. */
-		udo = (unsigned long)dictoffset;
-		udo = ((udo + PTRSIZE-1) / PTRSIZE) * PTRSIZE;
-		dictoffset = (long)udo;
+		size_t size;
+		_PyObject_VAR_SIZE(size, tp, ((PyVarObject *)obj)->ob_size);
+		dictoffset += (long)size;
+		assert(dictoffset > 0);
+		assert(dictoffset % SIZEOF_VOID_P == 0);
 	}
 	return (PyObject **) ((char *)obj + dictoffset);
 }

Objects/typeobject.c

@@ -190,28 +190,13 @@ type_call(PyTypeObject *type, PyObject *args, PyObject *kwds)
 PyObject *
 PyType_GenericAlloc(PyTypeObject *type, int nitems)
 {
-#define PTRSIZE (sizeof(PyObject *))
-
-	size_t size = (size_t)_PyObject_VAR_SIZE(type, nitems);
-	size_t padding = 0;
 	PyObject *obj;
+	size_t size;
 
-	/* Round up size, if necessary, so that the __dict__ pointer
-	   following the variable part is properly aligned for the platform.
-	   This is needed only for types with a vrbl number of items
-	   before the __dict__ pointer == types that record the dict offset
-	   as a negative offset from the end of the object.  If tp_dictoffset
-	   is 0, there is no __dict__; if positive, tp_dict was declared in a C
-	   struct so the compiler already took care of aligning it. */
-        if (type->tp_dictoffset < 0) {
-		padding = PTRSIZE - size % PTRSIZE;
-		if (padding == PTRSIZE)
-			padding = 0;
-		size += padding;
-	}
+	_PyObject_VAR_SIZE(size, type, nitems);
 
 	if (PyType_IS_GC(type))
-		obj = _PyObject_GC_Malloc(type, nitems, padding);
+		obj = _PyObject_GC_Malloc(type, nitems);
 	else
 		obj = PyObject_MALLOC(size);