Stackless issue #166: Fix PyEval_EvalFrameEx()

The C-API functions PyEval_EvalFrameEx() and PyEval_EvalFrame() were
broken, because Stackless does not use them and didn't test them. This
commit adds test code and makes the functions compatible with C-Python.
It is now save to call them from extension modules, i.e. modules created
by Cython.
(cherry picked from commit d377c06)

Author: Anselm Kruis

Python/ceval.c

@@ -3969,7 +3969,60 @@ slp_eval_frame_with_cleanup(PyFrameObject *f, int throwflag, PyObject *retval)
 PyObject * _Py_HOT_FUNCTION
 _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag)
 {
-    return PyEval_EvalFrameEx_slp(f, throwflag, NULL);
+    /*
+     * This method is not used by Stackless Python. It is provided for compatibility
+     * with extension modules and Cython.
+     */
+    PyThreadState *tstate = PyThreadState_GET();
+    PyObject * retval = NULL;
+
+    if (f == NULL)
+        return NULL;
+    /* make sure that the Stackless system has been initialized. */
+    assert(tstate->st.main && tstate->st.current);
+    /* sanity check. */
+    assert(SLP_CURRENT_FRAME_IS_VALID(tstate));
+    if (!throwflag) {
+        /* If throwflag is true, retval must be NULL. Otherwise it must be non-NULL.
+         */
+        Py_INCREF(Py_None);
+        retval = Py_None;
+    }
+    if (PyFrame_Check(f) && f->f_execute == NULL) {
+        /* A new frame returned from PyFrame_New() has f->f_execute == NULL.
+         */
+        f->f_execute = PyEval_EvalFrameEx_slp;
+    } else {
+        /* The layout of PyFrameObject differs between Stackless and C-Python.
+         * Stackless f->f_execute is C-Python f->f_code. Stackless f->f_code is at
+         * the end, just before f_localsplus.
+         *
+         * In order to detect a C-Python frame, we must compare f->f_execute
+         * with every valid frame function. Hard to implement completely.
+         * Therefore I'll check only for relevant functions.
+         * Amend the list as needed.
+         *
+         * If needed, we could try to fix an C-Python frame on the fly.
+         *
+         * (It is not possible to detect an C-Python frame by its size, because
+         * we need the code object to compute the expected size and the location
+         * of code objects varies between Stackless and C-Python frames).
+         */
+        if (!PyCFrame_Check(f) &&
+                f->f_execute != PyEval_EvalFrameEx_slp &&
+                f->f_execute != slp_eval_frame_value &&
+                f->f_execute != slp_eval_frame_noval &&
+                f->f_execute != slp_eval_frame_iter &&
+                f->f_execute != slp_eval_frame_setup_with &&
+                f->f_execute != slp_eval_frame_with_cleanup)  {
+            PyErr_BadInternalCall();
+            return NULL;
+        }
+    }
+    retval = slp_frame_dispatch(f, f->f_back, throwflag, retval);
+    assert(!STACKLESS_UNWINDING(retval));
+    assert(SLP_CURRENT_FRAME_IS_VALID(tstate));
+    return retval;
 }
 
 PyObject * _Py_HOT_FUNCTION

Stackless/changelog.txt

@@ -9,6 +9,10 @@ What's New in Stackless 3.X.X?
 
 *Release date: 20XX-XX-XX*
 
+- https://github.com/stackless-dev/stackless/issues/166
+  Fix C-API functions PyEval_EvalFrameEx() and PyEval_EvalFrame().
+  They are now compatible with C-Python.
+
 - https://github.com/stackless-dev/stackless/issues/167
   Replace 'printf(...)' calls by PySys_WriteStderr(...). They are used to emit
   an error message, if there is a pending error while entering Stackless

Stackless/module/stacklessmodule.c

@@ -1123,6 +1123,74 @@ static PyObject *get_test_nostacklesscallobj(void)
     return (PyObject*)PyObject_New(test_nostacklesscallObject, &test_nostacklesscallType);
 }
 
+
+PyDoc_STRVAR(test_PyEval_EvalFrameEx__doc__,
+"test_PyEval_EvalFrameEx(code, globals, args=()) -- a builtin testing function.\n\
+This function tests the C-API function PyEval_EvalFrameEx(), which is not used\n\
+by Stackless Python.\n\
+The function creates a frame from code, globals and args and executes the frame.");
+
+static PyObject* test_PyEval_EvalFrameEx(PyObject *self, PyObject *args, PyObject *kwds) {
+    static char *kwlist[] = {"code", "globals", "args", "alloca", NULL};
+    PyThreadState *tstate = PyThreadState_GET();
+    PyCodeObject *co;
+    PyObject *globals, *co_args = NULL;
+    Py_ssize_t alloca_size = 0;
+    PyFrameObject *f;
+    PyObject *result = NULL;
+    void *p;
+    Py_ssize_t na;
+
+    if (!PyArg_ParseTupleAndKeywords(args, kwds, "O!O!|O!n:test_PyEval_EvalFrameEx", kwlist,
+            &PyCode_Type, &co, &PyDict_Type, &globals, &PyTuple_Type, &co_args, &alloca_size))
+        return NULL;
+    p = alloca(alloca_size);
+    assert(globals != NULL);
+    assert(tstate != NULL);
+    na = PyTuple_Size(co->co_freevars);
+    if (na == -1)
+        return NULL;
+    if (na > 0) {
+        PyErr_Format(PyExc_ValueError, "code requires cell variables");
+        return NULL;
+    }
+    f = PyFrame_New(tstate, co, globals, NULL);
+    if (f == NULL) {
+        return NULL;
+    }
+    if (co_args) {
+        PyObject **fastlocals;
+        Py_ssize_t i;
+        na = PyTuple_Size(co_args);
+        if (na == -1)
+            goto exit;
+        if (na > co->co_argcount) {
+            PyErr_Format(PyExc_ValueError, "too many items in tuple 'args'");
+            goto exit;
+        }
+        fastlocals = f->f_localsplus;
+        for (i = 0; i < na; i++) {
+            PyObject *arg = PyTuple_GetItem(co_args, i);
+            if (arg == NULL) {
+                goto exit;
+            }
+            Py_INCREF(arg);
+            fastlocals[i] = arg;
+        }
+        if (alloca_size > 0 && na > 0) {
+            Py_SETREF(fastlocals[0], PyLong_FromVoidPtr(p));
+        }
+    }
+    result = PyEval_EvalFrameEx(f,0);
+    /* result = Py_None; Py_INCREF(Py_None); */
+exit:
+    ++tstate->recursion_depth;
+    Py_DECREF(f);
+    --tstate->recursion_depth;
+    return result;
+}
+
+
 /******************************************************
 
   The Stackless External Interface
@@ -1594,6 +1662,8 @@ static PyMethodDef stackless_methods[] = {
     test_outside__doc__},
     {"test_cstate",                 (PCF)test_cstate,           METH_O,
     test_cstate__doc__},
+    {"test_PyEval_EvalFrameEx",     (PCF)test_PyEval_EvalFrameEx, METH_VARARGS | METH_KEYWORDS,
+    test_PyEval_EvalFrameEx__doc__},
     {"set_channel_callback",        (PCF)set_channel_callback,  METH_O,
      set_channel_callback__doc__},
     {"get_channel_callback",        (PCF)get_channel_callback,  METH_NOARGS,

Stackless/unittests/test_capi.py

@@ -0,0 +1,100 @@
+#
+# -*- coding: utf-8 -*-
+#
+#  Copyright 2018 science + computing ag
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""Test the (Stackless)-Python C-API
+
+Tests not relevant for pure Python code.
+"""
+
+from __future__ import print_function, absolute_import, division
+
+import inspect
+import stackless
+import sys
+import unittest
+
+from support import test_main  # @UnusedImport
+from support import StacklessTestCase
+
+
+class Test_PyEval_EvalFrameEx(StacklessTestCase):
+    @staticmethod
+    def function(arg):
+        return arg
+
+    def call_PyEval_EvalFrameEx(self, *args, **kw):
+        f = self.function
+        if inspect.ismethod(f):
+            f = f.__func__
+        return stackless.test_PyEval_EvalFrameEx(f.__code__, f.__globals__, args, **kw)
+
+    def test_free_vars(self):
+        # stackless.test_PyEval_EvalFrameEx can't handle code, that contains free variables.
+        x = None
+
+        def f():
+            return x
+        self.assertTupleEqual(f.__code__.co_freevars, ('x',))
+        self.assertRaises(ValueError, stackless.test_PyEval_EvalFrameEx, f.__code__, f.__globals__)
+
+    def test_0_args(self):
+        self.assertRaises(UnboundLocalError, self.call_PyEval_EvalFrameEx)
+
+    def test_1_args(self):
+        self.assertEqual(self.call_PyEval_EvalFrameEx(47110815), 47110815)
+
+    def test_2_args(self):
+        self.assertRaises(ValueError, self.call_PyEval_EvalFrameEx, 4711, None)
+
+    def test_cstack_spilling(self):
+        # Force stack spilling. 16384 is the value of CSTACK_WATERMARK from slp_platformselect.h
+        self.call_PyEval_EvalFrameEx(None, alloca=16384 * 8)
+
+    def test_stack_unwinding(self):
+        # Calling the __init__ method of a new-style class involves stack unwinding
+        class C(object):
+            def __init__(self):
+                self.initialized = True
+
+        def f(C):
+            return C()
+
+        c = stackless.test_PyEval_EvalFrameEx(f.__code__, f.__globals__, (C,))
+        self.assertIsInstance(c, C)
+        self.assertTrue(c.initialized)
+
+    def test_tasklet_switch(self):
+        # test a tasklet switch out of the running frame
+        self.other_done = False
+
+        def other_tasklet():
+            self.other_done = True
+
+        def f():
+            stackless.run()
+            return 666
+
+        stackless.tasklet(other_tasklet)()
+        result = stackless.test_PyEval_EvalFrameEx(f.__code__, f.__globals__)
+        self.assertTrue(self.other_done)
+        self.assertEqual(result, 666)
+
+
+if __name__ == "__main__":
+    if not sys.argv[1:]:
+        sys.argv.append('-v')
+    unittest.main()