From 26afdc6639a8c03113e4fefae1ff7298eeb161df Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 19 Jul 2017 15:59:36 +0200 Subject: [PATCH 01/82] POC implementation of __base_subclass__ --- Python/bltinmodule.c | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 6215a638c94b71..3ecf3149406bb9 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -53,7 +53,9 @@ builtin___build_class__(PyObject *self, PyObject **args, Py_ssize_t nargs, { PyObject *func, *name, *bases, *mkw, *meta, *winner, *prep, *ns; PyObject *cls = NULL, *cell = NULL; + PyObject *base_types; int isclass = 0; /* initialize to prevent gcc warning */ + int i, modified_bases = 0; if (nargs < 2) { PyErr_SetString(PyExc_TypeError, @@ -76,6 +78,30 @@ builtin___build_class__(PyObject *self, PyObject **args, Py_ssize_t nargs, if (bases == NULL) return NULL; + for (i = 2; i < nargs; i++){ + PyObject *base, *new_base; + base = args[i]; + if PyType_Check(base){ + continue; + } + new_base = PyObject_GetAttrString(base, "__base_subclass__"); + if (new_base == NULL) { + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_Clear(); + } + else { + Py_DECREF(bases); + return NULL; + } + } + else { + Py_INCREF(new_base); + PyTuple_SET_ITEM(bases, i - 2, new_base); + Py_DECREF(base); + modified_bases = 1; + } + } + if (kwnames == NULL) { meta = NULL; mkw = NULL; @@ -168,6 +194,10 @@ builtin___build_class__(PyObject *self, PyObject **args, Py_ssize_t nargs, NULL, 0, NULL, 0, NULL, 0, NULL, PyFunction_GET_CLOSURE(func)); if (cell != NULL) { + if (modified_bases){ + base_types = _PyStack_AsTupleSlice(args, nargs, 2, nargs); + PyMapping_SetItemString(ns, "__orig_bases__", base_types); + } PyObject *margs[3] = {name, bases, ns}; cls = _PyObject_FastCallDict(meta, margs, 3, mkw); if (cls != NULL && PyType_Check(cls) && PyCell_Check(cell)) { From 3285fb6898739dfb844412f04d7638136e39d747 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 20 Jul 2017 01:00:00 +0200 Subject: [PATCH 02/82] POC implementation of __class_getitem__ --- Objects/typeobject.c | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 2a8118b43c5a0a..3f6cc9721c4260 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -3488,6 +3488,32 @@ type_is_gc(PyTypeObject *type) return type->tp_flags & Py_TPFLAGS_HEAPTYPE; } +static PyObject * +generic_subscript(PyTypeObject *tp, PyObject *key) +{ + PyObject* stack[2] = {(PyObject *)tp, key}; + PyObject *meth = PyObject_GetAttrString((PyObject *)tp, "__class_getitem__"); + const char* msg = "'%.200s' object is not subscriptable"; + if (meth){ + return _PyObject_FastCall(meth, stack, 2); + } + else{ + if (PyErr_ExceptionMatches(PyExc_AttributeError)){ + PyErr_Clear(); + PyErr_Format(PyExc_TypeError, msg, ((PyObject *)tp)->ob_type->tp_name); + return NULL; + } + return NULL; + } + return NULL; +} + +static PyMappingMethods generic_as_mapping = { + NULL, /*mp_length*/ + (binaryfunc)generic_subscript, /*mp_subscript*/ + NULL, /*mp_ass_subscript*/ +}; + PyTypeObject PyType_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "type", /* tp_name */ @@ -3501,7 +3527,7 @@ PyTypeObject PyType_Type = { (reprfunc)type_repr, /* tp_repr */ 0, /* tp_as_number */ 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ + &generic_as_mapping, /* tp_as_mapping */ 0, /* tp_hash */ (ternaryfunc)type_call, /* tp_call */ 0, /* tp_str */ From 779f85dcba319311d788e8aa7e6b87c923bf9935 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 20 Jul 2017 02:01:52 +0200 Subject: [PATCH 03/82] Modify __base_subclass__ API to support dynamic evaluation and base removal --- Python/bltinmodule.c | 34 ++++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 3ecf3149406bb9..a7ad7edc8f2b6d 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -51,7 +51,7 @@ static PyObject * builtin___build_class__(PyObject *self, PyObject **args, Py_ssize_t nargs, PyObject *kwnames) { - PyObject *func, *name, *bases, *mkw, *meta, *winner, *prep, *ns; + PyObject *func, *name, *bases, *mkw, *meta, *winner, *prep, *ns, *new_bases; PyObject *cls = NULL, *cell = NULL; PyObject *base_types; int isclass = 0; /* initialize to prevent gcc warning */ @@ -79,13 +79,14 @@ builtin___build_class__(PyObject *self, PyObject **args, Py_ssize_t nargs, return NULL; for (i = 2; i < nargs; i++){ - PyObject *base, *new_base; + PyObject *base, *new_base, *new_base_meth; + PyObject* stack[1]; base = args[i]; if PyType_Check(base){ continue; } - new_base = PyObject_GetAttrString(base, "__base_subclass__"); - if (new_base == NULL) { + new_base_meth = PyObject_GetAttrString(base, "__base_subclass__"); + if (new_base_meth == NULL) { if (PyErr_ExceptionMatches(PyExc_AttributeError)) { PyErr_Clear(); } @@ -95,6 +96,8 @@ builtin___build_class__(PyObject *self, PyObject **args, Py_ssize_t nargs, } } else { + stack[0] = bases; + new_base = _PyObject_FastCall(new_base_meth, stack, 1); Py_INCREF(new_base); PyTuple_SET_ITEM(bases, i - 2, new_base); Py_DECREF(base); @@ -102,6 +105,29 @@ builtin___build_class__(PyObject *self, PyObject **args, Py_ssize_t nargs, } } + if (modified_bases){ + int ind, tot_nones = 0; + for (i = 0; i < nargs - 2; i++){ + if (PyTuple_GET_ITEM(bases, i) == Py_None){ + tot_nones++; + } + } + ind = 0; + /* Remove all None's from base classes */ + new_bases = PyTuple_New(nargs - 2 - tot_nones); + for (i = 0; i < nargs - 2; i++){ + PyObject* a_base; + a_base = PyTuple_GET_ITEM(bases, i); + if (a_base != Py_None){ + Py_INCREF(a_base); + PyTuple_SET_ITEM(new_bases, ind, a_base); + ind++; + } + } + Py_DECREF(bases); + bases = new_bases; + } + if (kwnames == NULL) { meta = NULL; mkw = NULL; From 5d5211d9172660913f5b1507183117d837c26ba3 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 20 Jul 2017 23:20:43 +0200 Subject: [PATCH 04/82] Make __base_subclass__ faster and safer --- Python/bltinmodule.c | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index a7ad7edc8f2b6d..ba78b1364dedd9 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -74,6 +74,7 @@ builtin___build_class__(PyObject *self, PyObject **args, Py_ssize_t nargs, "__build_class__: name is not a string"); return NULL; } + bases = _PyStack_AsTupleSlice(args, nargs, 2, nargs); if (bases == NULL) return NULL; @@ -96,31 +97,42 @@ builtin___build_class__(PyObject *self, PyObject **args, Py_ssize_t nargs, } } else { + if (!PyCallable_Check(new_base_meth)) { + PyErr_Format(PyExc_TypeError, + "attribute of type '%.200s' is not callable", + Py_TYPE(new_base_meth)->tp_name); + Py_DECREF(bases); + return NULL; + } stack[0] = bases; new_base = _PyObject_FastCall(new_base_meth, stack, 1); + if (new_base == NULL){ + Py_DECREF(bases); + return NULL; + } Py_INCREF(new_base); - PyTuple_SET_ITEM(bases, i - 2, new_base); - Py_DECREF(base); + args[i] = new_base; modified_bases = 1; } } if (modified_bases){ int ind, tot_nones = 0; - for (i = 0; i < nargs - 2; i++){ - if (PyTuple_GET_ITEM(bases, i) == Py_None){ + for (i = 2; i < nargs; i++){ + if (args[i] == Py_None){ tot_nones++; } } + ind = 0; /* Remove all None's from base classes */ new_bases = PyTuple_New(nargs - 2 - tot_nones); - for (i = 0; i < nargs - 2; i++){ - PyObject* a_base; - a_base = PyTuple_GET_ITEM(bases, i); - if (a_base != Py_None){ - Py_INCREF(a_base); - PyTuple_SET_ITEM(new_bases, ind, a_base); + for (i = 2; i < nargs; i++){ + PyObject* base; + base = args[i]; + if (base != Py_None){ + Py_INCREF(base); + PyTuple_SET_ITEM(new_bases, ind, base); ind++; } } From 41fa7e9ce186efb3953ce99d0094f71f38fdc18f Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 20 Jul 2017 23:59:25 +0200 Subject: [PATCH 05/82] Factor out base update in a separate helper --- Python/bltinmodule.c | 104 ++++++++++++++++++++++++++----------------- 1 file changed, 62 insertions(+), 42 deletions(-) diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index ba78b1364dedd9..c5652165931f06 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -46,39 +46,15 @@ _Py_IDENTIFIER(stderr); #include "clinic/bltinmodule.c.h" -/* AC: cannot convert yet, waiting for *args support */ -static PyObject * -builtin___build_class__(PyObject *self, PyObject **args, Py_ssize_t nargs, - PyObject *kwnames) +static PyObject* +update_bases(PyObject* bases, PyObject** args, int nargs, int* modified_bases) { - PyObject *func, *name, *bases, *mkw, *meta, *winner, *prep, *ns, *new_bases; - PyObject *cls = NULL, *cell = NULL; - PyObject *base_types; - int isclass = 0; /* initialize to prevent gcc warning */ - int i, modified_bases = 0; - - if (nargs < 2) { - PyErr_SetString(PyExc_TypeError, - "__build_class__: not enough arguments"); - return NULL; - } - func = args[0]; /* Better be callable */ - if (!PyFunction_Check(func)) { - PyErr_SetString(PyExc_TypeError, - "__build_class__: func must be a function"); - return NULL; - } - name = args[1]; - if (!PyUnicode_Check(name)) { - PyErr_SetString(PyExc_TypeError, - "__build_class__: name is not a string"); - return NULL; - } - - bases = _PyStack_AsTupleSlice(args, nargs, 2, nargs); - if (bases == NULL) - return NULL; + int i, ind, tot_nones; + PyObject *new_bases; + assert(PyTuple_Check(bases)); + /* We have a separate cycle to calculate replacements with the idea that + most cases we just scroll quickly though it and return original bases */ for (i = 2; i < nargs; i++){ PyObject *base, *new_base, *new_base_meth; PyObject* stack[1]; @@ -92,7 +68,6 @@ builtin___build_class__(PyObject *self, PyObject **args, Py_ssize_t nargs, PyErr_Clear(); } else { - Py_DECREF(bases); return NULL; } } @@ -101,32 +76,31 @@ builtin___build_class__(PyObject *self, PyObject **args, Py_ssize_t nargs, PyErr_Format(PyExc_TypeError, "attribute of type '%.200s' is not callable", Py_TYPE(new_base_meth)->tp_name); - Py_DECREF(bases); return NULL; } stack[0] = bases; new_base = _PyObject_FastCall(new_base_meth, stack, 1); if (new_base == NULL){ - Py_DECREF(bases); return NULL; } Py_INCREF(new_base); args[i] = new_base; - modified_bases = 1; + *modified_bases = 1; } } - if (modified_bases){ - int ind, tot_nones = 0; + if (*modified_bases){ + /* Find out have many bases wants to be removed to pre-allocate + the tuple for new bases */ + tot_nones = 0; for (i = 2; i < nargs; i++){ if (args[i] == Py_None){ tot_nones++; } } - - ind = 0; - /* Remove all None's from base classes */ new_bases = PyTuple_New(nargs - 2 - tot_nones); + /* Remove all None's from base classes */ + ind = 0; for (i = 2; i < nargs; i++){ PyObject* base; base = args[i]; @@ -136,7 +110,54 @@ builtin___build_class__(PyObject *self, PyObject **args, Py_ssize_t nargs, ind++; } } + return new_bases; + } + else{ + return bases; + } +} + + +/* AC: cannot convert yet, waiting for *args support */ +static PyObject * +builtin___build_class__(PyObject *self, PyObject **args, Py_ssize_t nargs, + PyObject *kwnames) +{ + PyObject *func, *name, *bases, *mkw, *meta, *winner, *prep, *ns; + PyObject *new_bases, *old_bases = NULL; + PyObject *cls = NULL, *cell = NULL; + int isclass = 0; /* initialize to prevent gcc warning */ + int modified_bases = 0; + + if (nargs < 2) { + PyErr_SetString(PyExc_TypeError, + "__build_class__: not enough arguments"); + return NULL; + } + func = args[0]; /* Better be callable */ + if (!PyFunction_Check(func)) { + PyErr_SetString(PyExc_TypeError, + "__build_class__: func must be a function"); + return NULL; + } + name = args[1]; + if (!PyUnicode_Check(name)) { + PyErr_SetString(PyExc_TypeError, + "__build_class__: name is not a string"); + return NULL; + } + + bases = _PyStack_AsTupleSlice(args, nargs, 2, nargs); + if (bases == NULL) + return NULL; + + new_bases = update_bases(bases, args, nargs, &modified_bases); + if (new_bases == NULL){ Py_DECREF(bases); + return NULL; + } + else{ + old_bases = bases; bases = new_bases; } @@ -233,8 +254,7 @@ builtin___build_class__(PyObject *self, PyObject **args, Py_ssize_t nargs, PyFunction_GET_CLOSURE(func)); if (cell != NULL) { if (modified_bases){ - base_types = _PyStack_AsTupleSlice(args, nargs, 2, nargs); - PyMapping_SetItemString(ns, "__orig_bases__", base_types); + PyMapping_SetItemString(ns, "__orig_bases__", old_bases); } PyObject *margs[3] = {name, bases, ns}; cls = _PyObject_FastCallDict(meta, margs, 3, mkw); From 5aeebab8950d5efc3f2c6c36c3e1e7a78e7d146d Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 21 Jul 2017 00:05:12 +0200 Subject: [PATCH 06/82] Formatting --- Python/bltinmodule.c | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index c5652165931f06..1d5da34962fadf 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -59,7 +59,7 @@ update_bases(PyObject* bases, PyObject** args, int nargs, int* modified_bases) PyObject *base, *new_base, *new_base_meth; PyObject* stack[1]; base = args[i]; - if PyType_Check(base){ + if (PyType_Check(base)) { continue; } new_base_meth = PyObject_GetAttrString(base, "__base_subclass__"); @@ -93,18 +93,18 @@ update_bases(PyObject* bases, PyObject** args, int nargs, int* modified_bases) /* Find out have many bases wants to be removed to pre-allocate the tuple for new bases */ tot_nones = 0; - for (i = 2; i < nargs; i++){ - if (args[i] == Py_None){ + for (i = 2; i < nargs; i++) { + if (args[i] == Py_None) { tot_nones++; } } new_bases = PyTuple_New(nargs - 2 - tot_nones); /* Remove all None's from base classes */ ind = 0; - for (i = 2; i < nargs; i++){ + for (i = 2; i < nargs; i++) { PyObject* base; base = args[i]; - if (base != Py_None){ + if (base != Py_None) { Py_INCREF(base); PyTuple_SET_ITEM(new_bases, ind, base); ind++; @@ -112,7 +112,7 @@ update_bases(PyObject* bases, PyObject** args, int nargs, int* modified_bases) } return new_bases; } - else{ + else { return bases; } } @@ -146,17 +146,16 @@ builtin___build_class__(PyObject *self, PyObject **args, Py_ssize_t nargs, "__build_class__: name is not a string"); return NULL; } - bases = _PyStack_AsTupleSlice(args, nargs, 2, nargs); if (bases == NULL) return NULL; new_bases = update_bases(bases, args, nargs, &modified_bases); - if (new_bases == NULL){ + if (new_bases == NULL) { Py_DECREF(bases); return NULL; } - else{ + else { old_bases = bases; bases = new_bases; } From e858ea7501b6bf0ccee2a4afd3104c8b7b6da7bc Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 21 Jul 2017 00:41:46 +0200 Subject: [PATCH 07/82] Also make __class_getitem__ safer --- Objects/typeobject.c | 13 +++++++++---- Python/bltinmodule.c | 7 +++---- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 3f6cc9721c4260..9215a9f5a3c2af 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -3492,13 +3492,18 @@ static PyObject * generic_subscript(PyTypeObject *tp, PyObject *key) { PyObject* stack[2] = {(PyObject *)tp, key}; - PyObject *meth = PyObject_GetAttrString((PyObject *)tp, "__class_getitem__"); const char* msg = "'%.200s' object is not subscriptable"; - if (meth){ + PyObject *meth = PyObject_GetAttrString((PyObject *)tp, "__class_getitem__"); + if (meth) { + if (!PyCallable_Check(meth)) { + PyErr_SetString(PyExc_TypeError, + "__class_getitem__ must be callable"); + return NULL; + } return _PyObject_FastCall(meth, stack, 2); } - else{ - if (PyErr_ExceptionMatches(PyExc_AttributeError)){ + else { + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { PyErr_Clear(); PyErr_Format(PyExc_TypeError, msg, ((PyObject *)tp)->ob_type->tp_name); return NULL; diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 1d5da34962fadf..49b2a9a368b4b9 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -53,7 +53,7 @@ update_bases(PyObject* bases, PyObject** args, int nargs, int* modified_bases) PyObject *new_bases; assert(PyTuple_Check(bases)); - /* We have a separate cycle to calculate replacements with the idea that + /* We have a separate cycle to calculate replacements with the idea that in most cases we just scroll quickly though it and return original bases */ for (i = 2; i < nargs; i++){ PyObject *base, *new_base, *new_base_meth; @@ -73,9 +73,8 @@ update_bases(PyObject* bases, PyObject** args, int nargs, int* modified_bases) } else { if (!PyCallable_Check(new_base_meth)) { - PyErr_Format(PyExc_TypeError, - "attribute of type '%.200s' is not callable", - Py_TYPE(new_base_meth)->tp_name); + PyErr_SetString(PyExc_TypeError, + "__base_subclass__ must be callable"); return NULL; } stack[0] = bases; From 5b8d45367b2adb0015c37e7d592f00bba32d973a Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 21 Jul 2017 15:47:47 +0200 Subject: [PATCH 08/82] Simplify some code --- Objects/typeobject.c | 1 - Python/bltinmodule.c | 81 +++++++++++++++++++------------------------- 2 files changed, 35 insertions(+), 47 deletions(-) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 9215a9f5a3c2af..66fb921c3c615e 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -3510,7 +3510,6 @@ generic_subscript(PyTypeObject *tp, PyObject *key) } return NULL; } - return NULL; } static PyMappingMethods generic_as_mapping = { diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 49b2a9a368b4b9..215a2059b518f3 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -50,73 +50,62 @@ static PyObject* update_bases(PyObject* bases, PyObject** args, int nargs, int* modified_bases) { int i, ind, tot_nones; - PyObject *new_bases; + PyObject *base, *new_base, *new_base_meth, *new_bases; + PyObject* stack[1] = {bases}; assert(PyTuple_Check(bases)); /* We have a separate cycle to calculate replacements with the idea that in most cases we just scroll quickly though it and return original bases */ for (i = 2; i < nargs; i++){ - PyObject *base, *new_base, *new_base_meth; - PyObject* stack[1]; base = args[i]; if (PyType_Check(base)) { continue; } new_base_meth = PyObject_GetAttrString(base, "__base_subclass__"); - if (new_base_meth == NULL) { + if (!new_base_meth) { if (PyErr_ExceptionMatches(PyExc_AttributeError)) { PyErr_Clear(); + continue; } - else { - return NULL; - } - } - else { - if (!PyCallable_Check(new_base_meth)) { - PyErr_SetString(PyExc_TypeError, - "__base_subclass__ must be callable"); - return NULL; - } - stack[0] = bases; - new_base = _PyObject_FastCall(new_base_meth, stack, 1); - if (new_base == NULL){ - return NULL; - } - Py_INCREF(new_base); - args[i] = new_base; - *modified_bases = 1; + return NULL; } - } - - if (*modified_bases){ - /* Find out have many bases wants to be removed to pre-allocate - the tuple for new bases */ - tot_nones = 0; - for (i = 2; i < nargs; i++) { - if (args[i] == Py_None) { - tot_nones++; - } + if (!PyCallable_Check(new_base_meth)) { + PyErr_SetString(PyExc_TypeError, + "__base_subclass__ must be callable"); + return NULL; } - new_bases = PyTuple_New(nargs - 2 - tot_nones); - /* Remove all None's from base classes */ - ind = 0; - for (i = 2; i < nargs; i++) { - PyObject* base; - base = args[i]; - if (base != Py_None) { - Py_INCREF(base); - PyTuple_SET_ITEM(new_bases, ind, base); - ind++; - } + new_base = _PyObject_FastCall(new_base_meth, stack, 1); + if (!new_base){ + return NULL; } - return new_bases; + Py_INCREF(new_base); + args[i] = new_base; + *modified_bases = 1; } - else { + if (!*modified_bases){ return bases; } + /* Find out have many bases wants to be removed to pre-allocate + the tuple for new bases, then keep only non-None's in bases.*/ + tot_nones = 0; + for (i = 2; i < nargs; i++) { + if (args[i] == Py_None) { + tot_nones++; + } + } + new_bases = PyTuple_New(nargs - 2 - tot_nones); + ind = 0; + for (i = 2; i < nargs; i++) { + base = args[i]; + if (base != Py_None) { + Py_INCREF(base); + PyTuple_SET_ITEM(new_bases, ind, base); + ind++; + } + } + return new_bases; } - /* AC: cannot convert yet, waiting for *args support */ static PyObject * builtin___build_class__(PyObject *self, PyObject **args, Py_ssize_t nargs, From b3e52f12341fb840811cddd03c97e4bfe53ac8fe Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 22 Jul 2017 19:49:09 +0200 Subject: [PATCH 09/82] Initial work on typing2 --- Lib/typing2.py | 1347 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1347 insertions(+) create mode 100644 Lib/typing2.py diff --git a/Lib/typing2.py b/Lib/typing2.py new file mode 100644 index 00000000000000..c504bf5ec318ff --- /dev/null +++ b/Lib/typing2.py @@ -0,0 +1,1347 @@ +import abc +from abc import abstractmethod, abstractproperty +import collections +import contextlib +import functools +import re as stdlib_re # Avoid confusion with the re we export. +import sys +import types +import collections.abc +try: + from types import WrapperDescriptorType, MethodWrapperType, MethodDescriptorType +except ImportError: + WrapperDescriptorType = type(object.__init__) + MethodWrapperType = type(object().__str__) + MethodDescriptorType = type(str.join) + + +# Please keep __all__ alphabetized within each category. +__all__ = [ + # Super-special typing primitives. + 'Any', + 'Callable', + 'ClassVar', + 'Generic', + 'Optional', + 'Tuple', + 'Type', + 'TypeVar', + 'Union', + + # ABCs (from collections.abc). + 'AbstractSet', # collections.abc.Set. + 'ByteString', + 'Container', + 'ContextManager', + 'Hashable', + 'ItemsView', + 'Iterable', + 'Iterator', + 'KeysView', + 'Mapping', + 'MappingView', + 'MutableMapping', + 'MutableSequence', + 'MutableSet', + 'Sequence', + 'Sized', + 'ValuesView', + 'Awaitable', + 'AsyncIterator', + 'AsyncIterable', + 'Coroutine', + 'Collection', + 'AsyncGenerator', + #AsyncContextManager + + # Structural checks, a.k.a. protocols. + 'Reversible', + 'SupportsAbs', + 'SupportsBytes', + 'SupportsComplex', + 'SupportsFloat', + 'SupportsInt', + 'SupportsRound', + + # Concrete collection types. + 'Counter', + 'Deque', + 'Dict', + 'DefaultDict', + 'List', + 'Set', + 'FrozenSet', + 'NamedTuple', # Not really a type. + 'Generator', + + # One-off things. + 'AnyStr', + 'cast', + 'get_type_hints', + 'NewType', + 'no_type_check', + 'no_type_check_decorator', + 'overload', + 'Text', + 'TYPE_CHECKING', +] + +# The pseudo-submodules 're' and 'io' are part of the public +# namespace, but excluded from __all__ because they might stomp on +# legitimate imports of those modules. + +########################### +# Internal helper functions +########################### + +def _trim_name(nm): + whitelist = ('_TypingBase', '_FinalTypingBase') + if nm.startswith('_') and nm not in whitelist: + nm = nm[1:] + return nm + + +def _type_repr(obj): + """Return the repr() of an object, special-casing types (internal helper). + + If obj is a type, we return a shorter version than the default + type.__repr__, based on the module and qualified name, which is + typically enough to uniquely identify a type. For everything + else, we fall back on repr(obj). + """ + if isinstance(obj, type): + if obj.__module__ == 'builtins': + return obj.__qualname__ + qname = obj.__qualname__ + if obj.__module__ == 'typing2': + qname = _trim_name(qname) + return '%s.%s' % (obj.__module__, qname) + if obj is ...: + return('...') + if isinstance(obj, types.FunctionType): + return obj.__name__ + return repr(obj) + + +def _type_vars(types): + tvars = [] + for t in types: + if isinstance(t, _TypingBase): + t._get_type_vars(tvars) + elif isinstance(t, list): + tvars.extend(_type_vars(t)) + return tuple(tvars) + + +def _eval_type(t, globalns, localns): + if isinstance(t, _TypingBase): + return t._eval_type(globalns, localns) + return t + + +def _type_check(arg, msg): + """Check that the argument is a type, and return it (internal helper). + As a special case, accept None and return type(None) instead. + The msg argument is a human-readable error message, e.g. + + "Union[arg, ...]: arg should be a type." + + We append the repr() of the actual value (truncated to 100 chars). + """ + if arg is None: + return type(None) + if isinstance(arg, str): + arg = _ForwardRef(arg) + if ( + isinstance(arg, _TypingBase) and type(arg).__name__ == '_ClassVar' or + not isinstance(arg, (type, _TypingBase)) and not callable(arg) + ): + raise TypeError(msg + " Got %.100r." % (arg,)) + # Bare Union etc. are not valid as type arguments + if ( + type(arg).__name__ in ('_Union', '_Optional') and + not getattr(arg, '__origin__', None) or + isinstance(arg, _TypingBase) and arg in (Generic, Protocol) + ): + raise TypeError("Plain %s is not valid as type argument" % arg) + return arg + + +# Special typing constructs Union, Optional, Generic, Callable and Tuple +# use three special attributes for internal bookkeeping of generic types: +# * __parameters__ is a tuple of unique free type parameters of a generic +# type, for example, Dict[T, T].__parameters__ == (T,); +# * __origin__ keeps a reference to a type that was subscripted, +# e.g., Union[T, int].__origin__ == Union; +# * __args__ is a tuple of all arguments used in subscripting, +# e.g., Dict[T, int].__args__ == (T, int). + + +def _update_args(cls, args): + """An internal helper function: calculate substitution tree + for generic cls after replacing its type parameters with + substitutions in tvars -> args (if any). + Repeat the same following __origin__'s. + + Return a list of arguments with all possible substitutions + performed. Arguments that are generic classes themselves are represented + as tuples (so that no new classes are created by this function). + For example: _subs_tree(List[Tuple[int, T]][str]) == [(Tuple, int, str)] + """ + + return GenericType(cls.__origin__, args) + + +def _remove_dups_flatten(parameters): + """An internal helper for Union creation and substitution: flatten Union's + among parameters, then remove duplicates and strict subclasses. + """ + + # Flatten out Union[Union[...], ...]. + params = [] + for p in parameters: + if isinstance(p, _Union) and p.__origin__ is Union: + params.extend(p.__args__) + elif isinstance(p, tuple) and len(p) > 0 and p[0] is Union: + params.extend(p[1:]) + else: + params.append(p) + # Weed out strict duplicates, preserving the first of each occurrence. + all_params = set(params) + if len(all_params) < len(params): + new_params = [] + for t in params: + if t in all_params: + new_params.append(t) + all_params.remove(t) + params = new_params + assert not all_params, all_params + # Weed out subclasses. + # E.g. Union[int, Employee, Manager] == Union[int, Employee]. + # If object is present it will be sole survivor among proper classes. + # Never discard type variables. + # (In particular, Union[str, AnyStr] != AnyStr.) + all_params = set(params) + for t1 in params: + if not isinstance(t1, type): + continue + if any(isinstance(t2, type) and issubclass(t1, t2) + for t2 in all_params - {t1} + if not (isinstance(t2, GenericMeta) and + t2.__origin__ is not None)): + all_params.remove(t1) + return tuple(t for t in params if t in all_params) + + +def _check_generic(cls, parameters): + # Check correct count for parameters of a generic cls (internal helper). + if not cls.__parameters__: + raise TypeError("%s is not a generic class" % repr(cls)) + alen = len(parameters) + elen = len(cls.__parameters__) + if alen != elen: + raise TypeError("Too %s parameters for %s; actual %s, expected %s" % + ("many" if alen > elen else "few", repr(cls), alen, elen)) + + +########################################################### +# Special bases for everything: general, and for singletons +########################################################### + +class _TypingBase: + """Internal indicator of special typing constructs.""" + + __slots__ = ('__weakref__') + + def __new__(cls, *args, **kwds): + """Constructor. + + This only exists to give a better error message in case + someone tries to subclass a special typing object (not a good idea). + """ + if (len(args) == 3 and + isinstance(args[0], str) and + isinstance(args[1], tuple)): + # Close enough. + raise TypeError("Cannot subclass %r" % _type_repr(cls)) + return super().__new__(cls) + + def _eval_type(self, globalns, localns): + """Override this in subclasses to interpret forward references. + + For example, List['C'] is internally stored as + List[_ForwardRef('C')], which should evaluate to List[C], + where C is an object found in globalns or localns (searching + localns first, of course). + """ + return self + + def _get_type_vars(self, tvars): + pass + + def __repr__(self): + cls = type(self) + qname = _trim_name(cls.__qualname__) + return '%s.%s' % (cls.__module__, qname) + + def __call__(self, *args, **kwds): + raise TypeError("Cannot instantiate %r" % type(self)) + + def __instancecheck__(self, obj): + raise TypeError(f"{_type_repr(self)} cannot be used with isinstance().") + + def __subclasscheck__(self, cls): + raise TypeError(f"{_type_repr(self)} cannot be used with issubclass().") + + +class _FinalTypingBase(_TypingBase): + """Internal mix-in class to prevent instantiation. + + Prevents instantiation unless _root=True is given in class call. + It is used to create pseudo-singleton instances Any, Union, Optional, etc. + """ + + __slots__ = () + + def __new__(cls, *args, _root=False, **kwds): + self = super().__new__(cls, *args, **kwds) + if _root is True: + return self + raise TypeError("Cannot instantiate %r" % cls) + + def __reduce__(self): + return _trim_name(type(self).__name__) + + +######################################################################## +# Two non-singleton classes _ForwardRef (internal), and TypeVar (public) +######################################################################## + + +class _ForwardRef(_TypingBase): + """Internal wrapper to hold a forward reference.""" + + __slots__ = ('__forward_arg__', '__forward_code__', + '__forward_evaluated__', '__forward_value__') + + def __init__(self, arg): + super().__init__(arg) + if not isinstance(arg, str): + raise TypeError('Forward reference must be a string -- got %r' % (arg,)) + try: + code = compile(arg, '', 'eval') + except SyntaxError: + raise SyntaxError('Forward reference must be an expression -- got %r' % + (arg,)) + self.__forward_arg__ = arg + self.__forward_code__ = code + self.__forward_evaluated__ = False + self.__forward_value__ = None + + def _eval_type(self, globalns, localns): + if not self.__forward_evaluated__ or localns is not globalns: + if globalns is None and localns is None: + globalns = localns = {} + elif globalns is None: + globalns = localns + elif localns is None: + localns = globalns + self.__forward_value__ = _type_check( + eval(self.__forward_code__, globalns, localns), + "Forward references must evaluate to types.") + self.__forward_evaluated__ = True + return self.__forward_value__ + + def __eq__(self, other): + if not isinstance(other, _ForwardRef): + return NotImplemented + return (self.__forward_arg__ == other.__forward_arg__ and + self.__forward_value__ == other.__forward_value__) + + def __hash__(self): + return hash((self.__forward_arg__, self.__forward_value__)) + + def __repr__(self): + return '%r*' % (self.__forward_arg__,) + + +class TypeVar(_TypingBase): + """Type variable. + + Usage:: + + T = TypeVar('T') # Can be anything + A = TypeVar('A', str, bytes) # Must be str or bytes + + Type variables exist primarily for the benefit of static type + checkers. They serve as the parameters for generic types as well + as for generic function definitions. See class Generic for more + information on generic types. Generic functions work as follows: + + def repeat(x: T, n: int) -> List[T]: + '''Return a list containing n references to x.''' + return [x]*n + + def longest(x: A, y: A) -> A: + '''Return the longest of two strings.''' + return x if len(x) >= len(y) else y + + The latter example's signature is essentially the overloading + of (str, str) -> str and (bytes, bytes) -> bytes. Also note + that if the arguments are instances of some subclass of str, + the return type is still plain str. + + At runtime, isinstance(x, T) and issubclass(C, T) will raise TypeError. + + Type variables defined with covariant=True or contravariant=True + can be used do declare covariant or contravariant generic types. + See PEP 484 for more details. By default generic types are invariant + in all type variables. + + Type variables can be introspected. e.g.: + + T.__name__ == 'T' + T.__constraints__ == () + T.__covariant__ == False + T.__contravariant__ = False + A.__constraints__ == (str, bytes) + """ + + __slots__ = ('__name__', '__bound__', '__constraints__', + '__covariant__', '__contravariant__') + + def __init__(self, name, *constraints, bound=None, + covariant=False, contravariant=False): + self.__name__ = name + if covariant and contravariant: + raise ValueError("Bivariant types are not supported.") + self.__covariant__ = bool(covariant) + self.__contravariant__ = bool(contravariant) + if constraints and bound is not None: + raise TypeError("Constraints cannot be combined with bound=...") + if constraints and len(constraints) == 1: + raise TypeError("A single constraint is not allowed") + msg = "TypeVar(name, constraint, ...): constraints must be types." + self.__constraints__ = tuple(_type_check(t, msg) for t in constraints) + if bound: + self.__bound__ = _type_check(bound, "Bound must be a type.") + else: + self.__bound__ = None + + def _get_type_vars(self, tvars): + if self not in tvars: + tvars.append(self) + + def __repr__(self): + if self.__covariant__: + prefix = '+' + elif self.__contravariant__: + prefix = '-' + else: + prefix = '~' + return prefix + self.__name__ + +# Some unconstrained type variables. These are used by the container types. +# (These are not for export.) +T = TypeVar('T') # Any type. +KT = TypeVar('KT') # Key type. +VT = TypeVar('VT') # Value type. +T_co = TypeVar('T_co', covariant=True) # Any type covariant containers. +V_co = TypeVar('V_co', covariant=True) # Any type covariant containers. +VT_co = TypeVar('VT_co', covariant=True) # Value type covariant containers. +T_contra = TypeVar('T_contra', contravariant=True) # Ditto contravariant. + +# A useful type variable with constraints. This represents string types. +# (This one *is* for export!) +AnyStr = TypeVar('AnyStr', bytes, str) + +#################################################### +# Two non-subscriptable singletons: Any and NoReturn +#################################################### + +class _Any(_FinalTypingBase): + """Special type indicating an unconstrained type. + + - Any is compatible with every type. + - Any assumed to have all methods. + - All values assumed to be instances of Any. + + Note that all the above statements are true from the point of view of + static type checkers. At runtime, Any should not be used with instance + or class checks. + """ + + __slots__ = () + + +Any = _Any(_root=True) + + +class _NoReturn(_FinalTypingBase): + """Special type indicating functions that never return. + Example:: + + from typing import NoReturn + + def stop() -> NoReturn: + raise Exception('no way') + + This type is invalid in other positions, e.g., ``List[NoReturn]`` + will fail in static type checkers. + """ + + __slots__ = () + + +NoReturn = _NoReturn(_root=True) + +################################################################## +# The main class: represents subscriptions of all singletons below +################################################################## + +class GenericType(_TypingBase): + + __slots__ = ('__origin__', '__args__', '__parameters__', + '_name', '_call') + + def __init__(self, origin, args, name=None, call=True): + self.__origin__ = origin + self.__args__ = args + self.__parameters__ = () + self._call = call + if name: + self._name = name + elif isinstance(origin, type): + self._name = origin.__qualname__ + else: + self._name = repr(origin) + def __call__(self, *args, **kwargs): + if not self._call: + raise TypeError(f'{self} is not callable') + return self.__origin__(*args, **kwargs) + def __base_subclass__(self, bases): + return self.__origin__ + def __getitem__(self, args): + return _update_args(self, args) + def _eval_type(self): + return self + def __repr__(self): + args = ', '.join([_type_repr(a) for a in self.__args__]) + return self._name + f'[{args}]' + + +###################################################################### +# Subscriptable non-subclassable singletons: ClassVar, Union, Optional +###################################################################### + +class _ClassVar(_FinalTypingBase): + """Special type construct to mark class variables. + + An annotation wrapped in ClassVar indicates that a given + attribute is intended to be used as a class variable and + should not be set on instances of that class. Usage:: + + class Starship: + stats: ClassVar[Dict[str, int]] = {} # class variable + damage: int = 10 # instance variable + + ClassVar accepts only types and cannot be further subscribed. + + Note that ClassVar is not a class itself, and should not + be used with isinstance() or issubclass(). + """ + + __slots__ = () + + def __getitem__(self, item): + return GenericType(self, [item]) + + +ClassVar = _ClassVar(_root=True) + + +class _Union(_FinalTypingBase): + """Union type; Union[X, Y] means either X or Y. + + To define a union, use e.g. Union[int, str]. Details: + + - The arguments must be types and there must be at least one. + + - None as an argument is a special case and is replaced by + type(None). + + - Unions of unions are flattened, e.g.:: + + Union[Union[int, str], float] == Union[int, str, float] + + - Unions of a single argument vanish, e.g.:: + + Union[int] == int # The constructor actually returns int + + - Redundant arguments are skipped, e.g.:: + + Union[int, str, int] == Union[int, str] + + - When comparing unions, the argument order is ignored, e.g.:: + + Union[int, str] == Union[str, int] + + - When two arguments have a subclass relationship, the least + derived argument is kept, e.g.:: + + class Employee: pass + class Manager(Employee): pass + Union[int, Employee, Manager] == Union[int, Employee] + Union[Manager, int, Employee] == Union[int, Employee] + Union[Employee, Manager] == Employee + + - Similar for object:: + + Union[int, object] == object + + - You cannot subclass or instantiate a union. + + - You can use Optional[X] as a shorthand for Union[X, None]. + """ + + __slots__ = () + + def __getitem__(self, parameters): + if parameters == (): + raise TypeError("Cannot take a Union of no types.") + if not isinstance(parameters, tuple): + parameters = (parameters,) + msg = "Union[arg, ...]: each arg must be a type." + parameters = tuple(_type_check(p, msg) for p in parameters) + return GenericType(self, parameters) + +Union = _Union(_root=True) + + +class _Optional(_FinalTypingBase): + """Optional type. + + Optional[X] is equivalent to Union[X, None]. + """ + + __slots__ = () + + def __getitem__(self, arg): + arg = _type_check(arg, "Optional[t] requires a single type.") + return Union[arg, type(None)] + + +Optional = _Optional(_root=True) + + +############################################################# +# Subclassable singletons: Generic, Callable, Tuple, Protocol +############################################################# + +class Generic(_FinalTypingBase): + """Abstract base class for generic types. + + A generic type is typically declared by inheriting from + this class parameterized with one or more type variables. + For example, a generic mapping type might be defined as:: + + class Mapping(Generic[KT, VT]): + def __getitem__(self, key: KT) -> VT: + ... + # Etc. + + This class can then be used as follows:: + + def lookup_name(mapping: Mapping[KT, VT], key: KT, default: VT) -> VT: + try: + return mapping[key] + except KeyError: + return default + """ + + __slots__ = () + + def __class_getitem__(self, args): + return GenericType(self, args) + + +class Tuple(_FinalTypingBase): + """Tuple type; Tuple[X, Y] is the cross-product type of X and Y. + + Example: Tuple[T1, T2] is a tuple of two elements corresponding + to type variables T1 and T2. Tuple[int, float, str] is a tuple + of an int, a float and a string. + + To specify a variable-length tuple of homogeneous type, use Tuple[T, ...]. + """ + + __slots__ = () + + def __class_getitem__(self, args): + return GenericType(tuple, args, name='Tuple', call=False) + + +class Callable(_FinalTypingBase): + """Callable type; Callable[[int], str] is a function of (int) -> str. + + The subscription syntax must always be used with exactly two + values: the argument list and the return type. The argument list + must be a list of types or ellipsis; the return type must be a single type. + + There is no syntax to indicate optional or keyword arguments, + such function types are rarely used as callback types. + """ + + __slots__ = () + + def __class_getitem__(self, args): + return GenericType(collections.abc.Callable, args) + + +class Protocol(_FinalTypingBase): + """Internal base class for protocol classes. + + This implements a simple-minded structural issubclass check + (similar but more general than the one-offs in collections.abc + such as Hashable). + """ + + __slots__ = () + + def __class_getitem__(self, args): + return GenericType(self, args) + + +######################################### +# Fancy classes: NamedTuple and TypedDict +######################################### + +def _make_nmtuple(name, types): + msg = "NamedTuple('Name', [(f0, t0), (f1, t1), ...]); each t must be a type" + types = [(n, _type_check(t, msg)) for n, t in types] + nm_tpl = collections.namedtuple(name, [n for n, t in types]) + # Prior to PEP 526, only _field_types attribute was assigned. + # Now, both __annotations__ and _field_types are used to maintain compatibility. + nm_tpl.__annotations__ = nm_tpl._field_types = collections.OrderedDict(types) + try: + nm_tpl.__module__ = sys._getframe(2).f_globals.get('__name__', '__main__') + except (AttributeError, ValueError): + pass + return nm_tpl + + +_PY36 = sys.version_info[:2] >= (3, 6) + +# attributes prohibited to set in NamedTuple class syntax +_prohibited = ('__new__', '__init__', '__slots__', '__getnewargs__', + '_fields', '_field_defaults', '_field_types', + '_make', '_replace', '_asdict', '_source') + +_special = ('__module__', '__name__', '__qualname__', '__annotations__') + + +class NamedTupleMeta(type): + + def __new__(cls, typename, bases, ns): + if ns.get('_root', False): + return super().__new__(cls, typename, bases, ns) + if not _PY36: + raise TypeError("Class syntax for NamedTuple is only supported" + " in Python 3.6+") + types = ns.get('__annotations__', {}) + nm_tpl = _make_nmtuple(typename, types.items()) + defaults = [] + defaults_dict = {} + for field_name in types: + if field_name in ns: + default_value = ns[field_name] + defaults.append(default_value) + defaults_dict[field_name] = default_value + elif defaults: + raise TypeError("Non-default namedtuple field {field_name} cannot " + "follow default field(s) {default_names}" + .format(field_name=field_name, + default_names=', '.join(defaults_dict.keys()))) + nm_tpl.__new__.__defaults__ = tuple(defaults) + nm_tpl._field_defaults = defaults_dict + # update from user namespace without overriding special namedtuple attributes + for key in ns: + if key in _prohibited: + raise AttributeError("Cannot overwrite NamedTuple attribute " + key) + elif key not in _special and key not in nm_tpl._fields: + setattr(nm_tpl, key, ns[key]) + return nm_tpl + + +class NamedTuple(metaclass=NamedTupleMeta): + """Typed version of namedtuple. + + Usage in Python versions >= 3.6:: + + class Employee(NamedTuple): + name: str + id: int + + This is equivalent to:: + + Employee = collections.namedtuple('Employee', ['name', 'id']) + + The resulting class has extra __annotations__ and _field_types + attributes, giving an ordered dict mapping field names to types. + __annotations__ should be preferred, while _field_types + is kept to maintain pre PEP 526 compatibility. (The field names + are in the _fields attribute, which is part of the namedtuple + API.) Alternative equivalent keyword syntax is also accepted:: + + Employee = NamedTuple('Employee', name=str, id=int) + + In Python versions <= 3.5 use:: + + Employee = NamedTuple('Employee', [('name', str), ('id', int)]) + """ + _root = True + + def __new__(self, typename, fields=None, **kwargs): + if kwargs and not _PY36: + raise TypeError("Keyword syntax for NamedTuple is only supported" + " in Python 3.6+") + if fields is None: + fields = kwargs.items() + elif kwargs: + raise TypeError("Either list of fields or keywords" + " can be provided to NamedTuple, not both") + return _make_nmtuple(typename, fields) + + +################################# +# Public functions (with helpers) +################################# + +def cast(typ, val): + """Cast a value to a type. + + This returns the value unchanged. To the type checker this + signals that the return value has the designated type, but at + runtime we intentionally don't check anything (we want this + to be as fast as possible). + """ + return val + + +def _get_defaults(func): + """Internal helper to extract the default arguments, by name.""" + try: + code = func.__code__ + except AttributeError: + # Some built-in functions don't have __code__, __defaults__, etc. + return {} + pos_count = code.co_argcount + arg_names = code.co_varnames + arg_names = arg_names[:pos_count] + defaults = func.__defaults__ or () + kwdefaults = func.__kwdefaults__ + res = dict(kwdefaults) if kwdefaults else {} + pos_offset = pos_count - len(defaults) + for name, value in zip(arg_names[pos_offset:], defaults): + assert name not in res + res[name] = value + return res + + +_allowed_types = (types.FunctionType, types.BuiltinFunctionType, + types.MethodType, types.ModuleType, + WrapperDescriptorType, MethodWrapperType, MethodDescriptorType) + + +def get_type_hints(obj, globalns=None, localns=None): + """Return type hints for an object. + + This is often the same as obj.__annotations__, but it handles + forward references encoded as string literals, and if necessary + adds Optional[t] if a default value equal to None is set. + + The argument may be a module, class, method, or function. The annotations + are returned as a dictionary. For classes, annotations include also + inherited members. + + TypeError is raised if the argument is not of a type that can contain + annotations, and an empty dictionary is returned if no annotations are + present. + + BEWARE -- the behavior of globalns and localns is counterintuitive + (unless you are familiar with how eval() and exec() work). The + search order is locals first, then globals. + + - If no dict arguments are passed, an attempt is made to use the + globals from obj, and these are also used as the locals. If the + object does not appear to have globals, an exception is raised. + + - If one dict argument is passed, it is used for both globals and + locals. + + - If two dict arguments are passed, they specify globals and + locals, respectively. + """ + + if getattr(obj, '__no_type_check__', None): + return {} + if globalns is None: + globalns = getattr(obj, '__globals__', {}) + if localns is None: + localns = globalns + elif localns is None: + localns = globalns + # Classes require a special treatment. + if isinstance(obj, type): + hints = {} + for base in reversed(obj.__mro__): + ann = base.__dict__.get('__annotations__', {}) + for name, value in ann.items(): + if value is None: + value = type(None) + if isinstance(value, str): + value = _ForwardRef(value) + value = _eval_type(value, globalns, localns) + hints[name] = value + return hints + hints = getattr(obj, '__annotations__', None) + if hints is None: + # Return empty annotations for something that _could_ have them. + if isinstance(obj, _allowed_types): + return {} + else: + raise TypeError('{!r} is not a module, class, method, ' + 'or function.'.format(obj)) + defaults = _get_defaults(obj) + hints = dict(hints) + for name, value in hints.items(): + if value is None: + value = type(None) + if isinstance(value, str): + value = _ForwardRef(value) + value = _eval_type(value, globalns, localns) + if name in defaults and defaults[name] is None: + value = Optional[value] + hints[name] = value + return hints + + +def no_type_check(arg): + """Decorator to indicate that annotations are not type hints. + + The argument must be a class or function; if it is a class, it + applies recursively to all methods and classes defined in that class + (but not to methods defined in its superclasses or subclasses). + + This mutates the function(s) or class(es) in place. + """ + if isinstance(arg, type): + arg_attrs = arg.__dict__.copy() + for attr, val in arg.__dict__.items(): + if val in arg.__bases__ + (arg,): + arg_attrs.pop(attr) + for obj in arg_attrs.values(): + if isinstance(obj, types.FunctionType): + obj.__no_type_check__ = True + if isinstance(obj, type): + no_type_check(obj) + try: + arg.__no_type_check__ = True + except TypeError: # built-in classes + pass + return arg + + +def no_type_check_decorator(decorator): + """Decorator to give another decorator the @no_type_check effect. + + This wraps the decorator with something that wraps the decorated + function in @no_type_check. + """ + + @functools.wraps(decorator) + def wrapped_decorator(*args, **kwds): + func = decorator(*args, **kwds) + func = no_type_check(func) + return func + + return wrapped_decorator + + +def _overload_dummy(*args, **kwds): + """Helper for @overload to raise when called.""" + raise NotImplementedError( + "You should not call an overloaded function. " + "A series of @overload-decorated functions " + "outside a stub module should always be followed " + "by an implementation that is not @overload-ed.") + + +def overload(func): + """Decorator for overloaded functions/methods. + + In a stub file, place two or more stub definitions for the same + function in a row, each decorated with @overload. For example: + + @overload + def utf8(value: None) -> None: ... + @overload + def utf8(value: bytes) -> bytes: ... + @overload + def utf8(value: str) -> bytes: ... + + In a non-stub file (i.e. a regular .py file), do the same but + follow it with an implementation. The implementation should *not* + be decorated with @overload. For example: + + @overload + def utf8(value: None) -> None: ... + @overload + def utf8(value: bytes) -> bytes: ... + @overload + def utf8(value: str) -> bytes: ... + def utf8(value): + # implementation goes here + """ + return _overload_dummy + + +def NewType(name, tp): + """NewType creates simple unique types with almost zero + runtime overhead. NewType(name, tp) is considered a subtype of tp + by static type checkers. At runtime, NewType(name, tp) returns + a dummy function that simply returns its argument. Usage:: + + UserId = NewType('UserId', int) + + def name_by_id(user_id: UserId) -> str: + ... + + UserId('user') # Fails type check + + name_by_id(42) # Fails type check + name_by_id(UserId(42)) # OK + + num = UserId(5) + 1 # type: int + """ + + def new_type(x): + return x + + new_type.__name__ = name + new_type.__supertype__ = tp + return new_type + + +###################################################################### +# Various generics mimicking those in collections.abc and +# additional protocols. A few are simply re-exported for completeness. +###################################################################### + +Hashable = collections.abc.Hashable # Not generic. + +Awaitable = GenericType(collections.abc.Awaitable, [T_co]) +Coroutine = GenericType(collections.abc.Coroutine, [T_co, T_contra, V_co]) +AsyncIterable = GenericType(collections.abc.AsyncIterable, [T_co]) +AsyncIterator = GenericType(collections.abc.AsyncIterator, [T_co]) + +Iterable = GenericType(collections.abc.Iterable, [T_co]) +Iterator = GenericType(collections.abc.Iterator, [T_co]) + +class SupportsInt(Protocol): + __slots__ = () + + @abstractmethod + def __int__(self) -> int: + pass + + +class SupportsFloat(Protocol): + __slots__ = () + + @abstractmethod + def __float__(self) -> float: + pass + + +class SupportsComplex(Protocol): + __slots__ = () + + @abstractmethod + def __complex__(self) -> complex: + pass + + +class SupportsBytes(Protocol): + __slots__ = () + + @abstractmethod + def __bytes__(self) -> bytes: + pass + + +class SupportsAbs(Protocol[T_co]): + __slots__ = () + + @abstractmethod + def __abs__(self) -> T_co: + pass + + +class SupportsRound(Protocol[T_co]): + __slots__ = () + + @abstractmethod + def __round__(self, ndigits: int = 0) -> T_co: + pass + + +Reversible = GenericType(collections.abc.Reversible, [T_co]) +Sized = collections.abc.Sized # Not generic. +Container = GenericType(collections.abc.Container, [T_co]) +Collection = GenericType(collections.abc.Collection, [T_co]) + +# Callable was defined earlier. + +AbstractSet = GenericType(collections.abc.Set, [T_co]) +MutableSet = GenericType(collections.abc.MutableSet, [T]) +Set = GenericType(set, [T], name='Set') +FrozenSet = GenericType(frozenset, [T_co], name='FrozenSet') + +# NOTE: It is only covariant in the value type. +Mapping = GenericType(collections.abc.Mapping, [KT, VT_co]) +MutableMapping = GenericType(collections.abc.MutableMapping, [KT, VT]) +Dict = GenericType(dict, [KT, VT], call=False, name='Dict') +DefaultDict = GenericType(collections.defaultdict, [KT, VT], call=False, name='DefaultDict') +Counter = GenericType(collections.Counter, [T], call=False) +ChainMap = GenericType(collections.ChainMap, [KT, VT], call=False) + +Sequence = GenericType(collections.abc.Sequence, [T_co]) +MutableSequence = GenericType(collections.abc.MutableSequence, [T]) +ByteString = collections.abc.ByteString # Not generic. +List = GenericType(list, [T], name='List') +Deque = GenericType(collections.deque, [T], name='Deque') + +MappingView = GenericType(collections.abc.MappingView, [T_co]) +KeysView = GenericType(collections.abc.KeysView, [KT]) +ItemsView = GenericType(collections.abc.ItemsView, [KT, VT_co]) +ValuesView = GenericType(collections.abc.ValuesView, [VT_co]) + +ContextManager = GenericType(contextlib.AbstractContextManager, [T_co]) +if hasattr(contextlib, 'AbstractAsyncContextManager'): + AsyncContextManager = GenericType(contextlib.AbstractAsyncContextManager, [T_co]) + __all__.append('AsyncContextManager') + +Generator = GenericType(collections.abc.Generator, [T_co, T_contra, V_co]) +AsyncGenerator = GenericType(collections.abc.AsyncGenerator, [T_co, T_contra]) + +# Internal type variable used for Type[]. +CT_co = TypeVar('CT_co', covariant=True, bound=type) +# This is not a real generic class. Don't use outside annotations. +Type = GenericType(type, [CT_co], name='Type') +"""A special construct usable to annotate class objects. + +For example, suppose we have the following classes:: + + class User: ... # Abstract base for User classes + class BasicUser(User): ... + class ProUser(User): ... + class TeamUser(User): ... + +And a function that takes a class argument that's a subclass of +User and returns an instance of the corresponding class:: + + U = TypeVar('U', bound=User) + def new_user(user_class: Type[U]) -> U: + user = user_class() + # (Here we could write the user object to a database) + return user + + joe = new_user(BasicUser) + +At this point the type checker knows that joe has type BasicUser. +""" + +################## +# Public constants +################## + +# Python-version-specific alias (Python 2: unicode; Python 3: str) +Text = str + + +# Constant that's True when type checking, but False here. +TYPE_CHECKING = False + +######################################### +# Special io types and re generic aliases +######################################### + + +class IO(Generic[AnyStr]): + """Generic base class for TextIO and BinaryIO. + + This is an abstract, generic version of the return of open(). + + NOTE: This does not distinguish between the different possible + classes (text vs. binary, read vs. write vs. read/write, + append-only, unbuffered). The TextIO and BinaryIO subclasses + below capture the distinctions between text vs. binary, which is + pervasive in the interface; however we currently do not offer a + way to track the other distinctions in the type system. + """ + + __slots__ = () + + @abstractproperty + def mode(self) -> str: + pass + + @abstractproperty + def name(self) -> str: + pass + + @abstractmethod + def close(self) -> None: + pass + + @abstractmethod + def closed(self) -> bool: + pass + + @abstractmethod + def fileno(self) -> int: + pass + + @abstractmethod + def flush(self) -> None: + pass + + @abstractmethod + def isatty(self) -> bool: + pass + + @abstractmethod + def read(self, n: int = -1) -> AnyStr: + pass + + @abstractmethod + def readable(self) -> bool: + pass + + @abstractmethod + def readline(self, limit: int = -1) -> AnyStr: + pass + + @abstractmethod + def readlines(self, hint: int = -1) -> List[AnyStr]: + pass + + @abstractmethod + def seek(self, offset: int, whence: int = 0) -> int: + pass + + @abstractmethod + def seekable(self) -> bool: + pass + + @abstractmethod + def tell(self) -> int: + pass + + @abstractmethod + def truncate(self, size: int = None) -> int: + pass + + @abstractmethod + def writable(self) -> bool: + pass + + @abstractmethod + def write(self, s: AnyStr) -> int: + pass + + @abstractmethod + def writelines(self, lines: List[AnyStr]) -> None: + pass + + @abstractmethod + def __enter__(self) -> 'IO[AnyStr]': + pass + + @abstractmethod + def __exit__(self, type, value, traceback) -> None: + pass + + +class BinaryIO(IO[bytes]): + """Typed version of the return of open() in binary mode.""" + + __slots__ = () + + @abstractmethod + def write(self, s: Union[bytes, bytearray]) -> int: + pass + + @abstractmethod + def __enter__(self) -> 'BinaryIO': + pass + + +class TextIO(IO[str]): + """Typed version of the return of open() in text mode.""" + + __slots__ = () + + @abstractproperty + def buffer(self) -> BinaryIO: + pass + + @abstractproperty + def encoding(self) -> str: + pass + + @abstractproperty + def errors(self) -> Optional[str]: + pass + + @abstractproperty + def line_buffering(self) -> bool: + pass + + @abstractproperty + def newlines(self) -> Any: + pass + + @abstractmethod + def __enter__(self) -> 'TextIO': + pass + + +class io: + """Wrapper namespace for IO generic classes.""" + + __all__ = ['IO', 'TextIO', 'BinaryIO'] + IO = IO + TextIO = TextIO + BinaryIO = BinaryIO + + +io.__name__ = __name__ + '.io' +sys.modules[io.__name__] = io + + +Pattern = GenericType(type(stdlib_re.compile('')), [AnyStr], name='Pattern') +Match = GenericType(type(stdlib_re.match('', '')), [AnyStr], name='Match') + + +class re: + """Wrapper namespace for re type aliases.""" + + __all__ = ['Pattern', 'Match'] + Pattern = Pattern + Match = Match + + +re.__name__ = __name__ + '.re' +sys.modules[re.__name__] = re From 6e7b575273aed0a4c5f181fde1825d82d9924b8d Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 3 Sep 2017 20:57:35 +0200 Subject: [PATCH 10/82] Remove test implementation for typing --- Lib/typing2.py | 1347 ------------------------------------------------ 1 file changed, 1347 deletions(-) delete mode 100644 Lib/typing2.py diff --git a/Lib/typing2.py b/Lib/typing2.py deleted file mode 100644 index c504bf5ec318ff..00000000000000 --- a/Lib/typing2.py +++ /dev/null @@ -1,1347 +0,0 @@ -import abc -from abc import abstractmethod, abstractproperty -import collections -import contextlib -import functools -import re as stdlib_re # Avoid confusion with the re we export. -import sys -import types -import collections.abc -try: - from types import WrapperDescriptorType, MethodWrapperType, MethodDescriptorType -except ImportError: - WrapperDescriptorType = type(object.__init__) - MethodWrapperType = type(object().__str__) - MethodDescriptorType = type(str.join) - - -# Please keep __all__ alphabetized within each category. -__all__ = [ - # Super-special typing primitives. - 'Any', - 'Callable', - 'ClassVar', - 'Generic', - 'Optional', - 'Tuple', - 'Type', - 'TypeVar', - 'Union', - - # ABCs (from collections.abc). - 'AbstractSet', # collections.abc.Set. - 'ByteString', - 'Container', - 'ContextManager', - 'Hashable', - 'ItemsView', - 'Iterable', - 'Iterator', - 'KeysView', - 'Mapping', - 'MappingView', - 'MutableMapping', - 'MutableSequence', - 'MutableSet', - 'Sequence', - 'Sized', - 'ValuesView', - 'Awaitable', - 'AsyncIterator', - 'AsyncIterable', - 'Coroutine', - 'Collection', - 'AsyncGenerator', - #AsyncContextManager - - # Structural checks, a.k.a. protocols. - 'Reversible', - 'SupportsAbs', - 'SupportsBytes', - 'SupportsComplex', - 'SupportsFloat', - 'SupportsInt', - 'SupportsRound', - - # Concrete collection types. - 'Counter', - 'Deque', - 'Dict', - 'DefaultDict', - 'List', - 'Set', - 'FrozenSet', - 'NamedTuple', # Not really a type. - 'Generator', - - # One-off things. - 'AnyStr', - 'cast', - 'get_type_hints', - 'NewType', - 'no_type_check', - 'no_type_check_decorator', - 'overload', - 'Text', - 'TYPE_CHECKING', -] - -# The pseudo-submodules 're' and 'io' are part of the public -# namespace, but excluded from __all__ because they might stomp on -# legitimate imports of those modules. - -########################### -# Internal helper functions -########################### - -def _trim_name(nm): - whitelist = ('_TypingBase', '_FinalTypingBase') - if nm.startswith('_') and nm not in whitelist: - nm = nm[1:] - return nm - - -def _type_repr(obj): - """Return the repr() of an object, special-casing types (internal helper). - - If obj is a type, we return a shorter version than the default - type.__repr__, based on the module and qualified name, which is - typically enough to uniquely identify a type. For everything - else, we fall back on repr(obj). - """ - if isinstance(obj, type): - if obj.__module__ == 'builtins': - return obj.__qualname__ - qname = obj.__qualname__ - if obj.__module__ == 'typing2': - qname = _trim_name(qname) - return '%s.%s' % (obj.__module__, qname) - if obj is ...: - return('...') - if isinstance(obj, types.FunctionType): - return obj.__name__ - return repr(obj) - - -def _type_vars(types): - tvars = [] - for t in types: - if isinstance(t, _TypingBase): - t._get_type_vars(tvars) - elif isinstance(t, list): - tvars.extend(_type_vars(t)) - return tuple(tvars) - - -def _eval_type(t, globalns, localns): - if isinstance(t, _TypingBase): - return t._eval_type(globalns, localns) - return t - - -def _type_check(arg, msg): - """Check that the argument is a type, and return it (internal helper). - As a special case, accept None and return type(None) instead. - The msg argument is a human-readable error message, e.g. - - "Union[arg, ...]: arg should be a type." - - We append the repr() of the actual value (truncated to 100 chars). - """ - if arg is None: - return type(None) - if isinstance(arg, str): - arg = _ForwardRef(arg) - if ( - isinstance(arg, _TypingBase) and type(arg).__name__ == '_ClassVar' or - not isinstance(arg, (type, _TypingBase)) and not callable(arg) - ): - raise TypeError(msg + " Got %.100r." % (arg,)) - # Bare Union etc. are not valid as type arguments - if ( - type(arg).__name__ in ('_Union', '_Optional') and - not getattr(arg, '__origin__', None) or - isinstance(arg, _TypingBase) and arg in (Generic, Protocol) - ): - raise TypeError("Plain %s is not valid as type argument" % arg) - return arg - - -# Special typing constructs Union, Optional, Generic, Callable and Tuple -# use three special attributes for internal bookkeeping of generic types: -# * __parameters__ is a tuple of unique free type parameters of a generic -# type, for example, Dict[T, T].__parameters__ == (T,); -# * __origin__ keeps a reference to a type that was subscripted, -# e.g., Union[T, int].__origin__ == Union; -# * __args__ is a tuple of all arguments used in subscripting, -# e.g., Dict[T, int].__args__ == (T, int). - - -def _update_args(cls, args): - """An internal helper function: calculate substitution tree - for generic cls after replacing its type parameters with - substitutions in tvars -> args (if any). - Repeat the same following __origin__'s. - - Return a list of arguments with all possible substitutions - performed. Arguments that are generic classes themselves are represented - as tuples (so that no new classes are created by this function). - For example: _subs_tree(List[Tuple[int, T]][str]) == [(Tuple, int, str)] - """ - - return GenericType(cls.__origin__, args) - - -def _remove_dups_flatten(parameters): - """An internal helper for Union creation and substitution: flatten Union's - among parameters, then remove duplicates and strict subclasses. - """ - - # Flatten out Union[Union[...], ...]. - params = [] - for p in parameters: - if isinstance(p, _Union) and p.__origin__ is Union: - params.extend(p.__args__) - elif isinstance(p, tuple) and len(p) > 0 and p[0] is Union: - params.extend(p[1:]) - else: - params.append(p) - # Weed out strict duplicates, preserving the first of each occurrence. - all_params = set(params) - if len(all_params) < len(params): - new_params = [] - for t in params: - if t in all_params: - new_params.append(t) - all_params.remove(t) - params = new_params - assert not all_params, all_params - # Weed out subclasses. - # E.g. Union[int, Employee, Manager] == Union[int, Employee]. - # If object is present it will be sole survivor among proper classes. - # Never discard type variables. - # (In particular, Union[str, AnyStr] != AnyStr.) - all_params = set(params) - for t1 in params: - if not isinstance(t1, type): - continue - if any(isinstance(t2, type) and issubclass(t1, t2) - for t2 in all_params - {t1} - if not (isinstance(t2, GenericMeta) and - t2.__origin__ is not None)): - all_params.remove(t1) - return tuple(t for t in params if t in all_params) - - -def _check_generic(cls, parameters): - # Check correct count for parameters of a generic cls (internal helper). - if not cls.__parameters__: - raise TypeError("%s is not a generic class" % repr(cls)) - alen = len(parameters) - elen = len(cls.__parameters__) - if alen != elen: - raise TypeError("Too %s parameters for %s; actual %s, expected %s" % - ("many" if alen > elen else "few", repr(cls), alen, elen)) - - -########################################################### -# Special bases for everything: general, and for singletons -########################################################### - -class _TypingBase: - """Internal indicator of special typing constructs.""" - - __slots__ = ('__weakref__') - - def __new__(cls, *args, **kwds): - """Constructor. - - This only exists to give a better error message in case - someone tries to subclass a special typing object (not a good idea). - """ - if (len(args) == 3 and - isinstance(args[0], str) and - isinstance(args[1], tuple)): - # Close enough. - raise TypeError("Cannot subclass %r" % _type_repr(cls)) - return super().__new__(cls) - - def _eval_type(self, globalns, localns): - """Override this in subclasses to interpret forward references. - - For example, List['C'] is internally stored as - List[_ForwardRef('C')], which should evaluate to List[C], - where C is an object found in globalns or localns (searching - localns first, of course). - """ - return self - - def _get_type_vars(self, tvars): - pass - - def __repr__(self): - cls = type(self) - qname = _trim_name(cls.__qualname__) - return '%s.%s' % (cls.__module__, qname) - - def __call__(self, *args, **kwds): - raise TypeError("Cannot instantiate %r" % type(self)) - - def __instancecheck__(self, obj): - raise TypeError(f"{_type_repr(self)} cannot be used with isinstance().") - - def __subclasscheck__(self, cls): - raise TypeError(f"{_type_repr(self)} cannot be used with issubclass().") - - -class _FinalTypingBase(_TypingBase): - """Internal mix-in class to prevent instantiation. - - Prevents instantiation unless _root=True is given in class call. - It is used to create pseudo-singleton instances Any, Union, Optional, etc. - """ - - __slots__ = () - - def __new__(cls, *args, _root=False, **kwds): - self = super().__new__(cls, *args, **kwds) - if _root is True: - return self - raise TypeError("Cannot instantiate %r" % cls) - - def __reduce__(self): - return _trim_name(type(self).__name__) - - -######################################################################## -# Two non-singleton classes _ForwardRef (internal), and TypeVar (public) -######################################################################## - - -class _ForwardRef(_TypingBase): - """Internal wrapper to hold a forward reference.""" - - __slots__ = ('__forward_arg__', '__forward_code__', - '__forward_evaluated__', '__forward_value__') - - def __init__(self, arg): - super().__init__(arg) - if not isinstance(arg, str): - raise TypeError('Forward reference must be a string -- got %r' % (arg,)) - try: - code = compile(arg, '', 'eval') - except SyntaxError: - raise SyntaxError('Forward reference must be an expression -- got %r' % - (arg,)) - self.__forward_arg__ = arg - self.__forward_code__ = code - self.__forward_evaluated__ = False - self.__forward_value__ = None - - def _eval_type(self, globalns, localns): - if not self.__forward_evaluated__ or localns is not globalns: - if globalns is None and localns is None: - globalns = localns = {} - elif globalns is None: - globalns = localns - elif localns is None: - localns = globalns - self.__forward_value__ = _type_check( - eval(self.__forward_code__, globalns, localns), - "Forward references must evaluate to types.") - self.__forward_evaluated__ = True - return self.__forward_value__ - - def __eq__(self, other): - if not isinstance(other, _ForwardRef): - return NotImplemented - return (self.__forward_arg__ == other.__forward_arg__ and - self.__forward_value__ == other.__forward_value__) - - def __hash__(self): - return hash((self.__forward_arg__, self.__forward_value__)) - - def __repr__(self): - return '%r*' % (self.__forward_arg__,) - - -class TypeVar(_TypingBase): - """Type variable. - - Usage:: - - T = TypeVar('T') # Can be anything - A = TypeVar('A', str, bytes) # Must be str or bytes - - Type variables exist primarily for the benefit of static type - checkers. They serve as the parameters for generic types as well - as for generic function definitions. See class Generic for more - information on generic types. Generic functions work as follows: - - def repeat(x: T, n: int) -> List[T]: - '''Return a list containing n references to x.''' - return [x]*n - - def longest(x: A, y: A) -> A: - '''Return the longest of two strings.''' - return x if len(x) >= len(y) else y - - The latter example's signature is essentially the overloading - of (str, str) -> str and (bytes, bytes) -> bytes. Also note - that if the arguments are instances of some subclass of str, - the return type is still plain str. - - At runtime, isinstance(x, T) and issubclass(C, T) will raise TypeError. - - Type variables defined with covariant=True or contravariant=True - can be used do declare covariant or contravariant generic types. - See PEP 484 for more details. By default generic types are invariant - in all type variables. - - Type variables can be introspected. e.g.: - - T.__name__ == 'T' - T.__constraints__ == () - T.__covariant__ == False - T.__contravariant__ = False - A.__constraints__ == (str, bytes) - """ - - __slots__ = ('__name__', '__bound__', '__constraints__', - '__covariant__', '__contravariant__') - - def __init__(self, name, *constraints, bound=None, - covariant=False, contravariant=False): - self.__name__ = name - if covariant and contravariant: - raise ValueError("Bivariant types are not supported.") - self.__covariant__ = bool(covariant) - self.__contravariant__ = bool(contravariant) - if constraints and bound is not None: - raise TypeError("Constraints cannot be combined with bound=...") - if constraints and len(constraints) == 1: - raise TypeError("A single constraint is not allowed") - msg = "TypeVar(name, constraint, ...): constraints must be types." - self.__constraints__ = tuple(_type_check(t, msg) for t in constraints) - if bound: - self.__bound__ = _type_check(bound, "Bound must be a type.") - else: - self.__bound__ = None - - def _get_type_vars(self, tvars): - if self not in tvars: - tvars.append(self) - - def __repr__(self): - if self.__covariant__: - prefix = '+' - elif self.__contravariant__: - prefix = '-' - else: - prefix = '~' - return prefix + self.__name__ - -# Some unconstrained type variables. These are used by the container types. -# (These are not for export.) -T = TypeVar('T') # Any type. -KT = TypeVar('KT') # Key type. -VT = TypeVar('VT') # Value type. -T_co = TypeVar('T_co', covariant=True) # Any type covariant containers. -V_co = TypeVar('V_co', covariant=True) # Any type covariant containers. -VT_co = TypeVar('VT_co', covariant=True) # Value type covariant containers. -T_contra = TypeVar('T_contra', contravariant=True) # Ditto contravariant. - -# A useful type variable with constraints. This represents string types. -# (This one *is* for export!) -AnyStr = TypeVar('AnyStr', bytes, str) - -#################################################### -# Two non-subscriptable singletons: Any and NoReturn -#################################################### - -class _Any(_FinalTypingBase): - """Special type indicating an unconstrained type. - - - Any is compatible with every type. - - Any assumed to have all methods. - - All values assumed to be instances of Any. - - Note that all the above statements are true from the point of view of - static type checkers. At runtime, Any should not be used with instance - or class checks. - """ - - __slots__ = () - - -Any = _Any(_root=True) - - -class _NoReturn(_FinalTypingBase): - """Special type indicating functions that never return. - Example:: - - from typing import NoReturn - - def stop() -> NoReturn: - raise Exception('no way') - - This type is invalid in other positions, e.g., ``List[NoReturn]`` - will fail in static type checkers. - """ - - __slots__ = () - - -NoReturn = _NoReturn(_root=True) - -################################################################## -# The main class: represents subscriptions of all singletons below -################################################################## - -class GenericType(_TypingBase): - - __slots__ = ('__origin__', '__args__', '__parameters__', - '_name', '_call') - - def __init__(self, origin, args, name=None, call=True): - self.__origin__ = origin - self.__args__ = args - self.__parameters__ = () - self._call = call - if name: - self._name = name - elif isinstance(origin, type): - self._name = origin.__qualname__ - else: - self._name = repr(origin) - def __call__(self, *args, **kwargs): - if not self._call: - raise TypeError(f'{self} is not callable') - return self.__origin__(*args, **kwargs) - def __base_subclass__(self, bases): - return self.__origin__ - def __getitem__(self, args): - return _update_args(self, args) - def _eval_type(self): - return self - def __repr__(self): - args = ', '.join([_type_repr(a) for a in self.__args__]) - return self._name + f'[{args}]' - - -###################################################################### -# Subscriptable non-subclassable singletons: ClassVar, Union, Optional -###################################################################### - -class _ClassVar(_FinalTypingBase): - """Special type construct to mark class variables. - - An annotation wrapped in ClassVar indicates that a given - attribute is intended to be used as a class variable and - should not be set on instances of that class. Usage:: - - class Starship: - stats: ClassVar[Dict[str, int]] = {} # class variable - damage: int = 10 # instance variable - - ClassVar accepts only types and cannot be further subscribed. - - Note that ClassVar is not a class itself, and should not - be used with isinstance() or issubclass(). - """ - - __slots__ = () - - def __getitem__(self, item): - return GenericType(self, [item]) - - -ClassVar = _ClassVar(_root=True) - - -class _Union(_FinalTypingBase): - """Union type; Union[X, Y] means either X or Y. - - To define a union, use e.g. Union[int, str]. Details: - - - The arguments must be types and there must be at least one. - - - None as an argument is a special case and is replaced by - type(None). - - - Unions of unions are flattened, e.g.:: - - Union[Union[int, str], float] == Union[int, str, float] - - - Unions of a single argument vanish, e.g.:: - - Union[int] == int # The constructor actually returns int - - - Redundant arguments are skipped, e.g.:: - - Union[int, str, int] == Union[int, str] - - - When comparing unions, the argument order is ignored, e.g.:: - - Union[int, str] == Union[str, int] - - - When two arguments have a subclass relationship, the least - derived argument is kept, e.g.:: - - class Employee: pass - class Manager(Employee): pass - Union[int, Employee, Manager] == Union[int, Employee] - Union[Manager, int, Employee] == Union[int, Employee] - Union[Employee, Manager] == Employee - - - Similar for object:: - - Union[int, object] == object - - - You cannot subclass or instantiate a union. - - - You can use Optional[X] as a shorthand for Union[X, None]. - """ - - __slots__ = () - - def __getitem__(self, parameters): - if parameters == (): - raise TypeError("Cannot take a Union of no types.") - if not isinstance(parameters, tuple): - parameters = (parameters,) - msg = "Union[arg, ...]: each arg must be a type." - parameters = tuple(_type_check(p, msg) for p in parameters) - return GenericType(self, parameters) - -Union = _Union(_root=True) - - -class _Optional(_FinalTypingBase): - """Optional type. - - Optional[X] is equivalent to Union[X, None]. - """ - - __slots__ = () - - def __getitem__(self, arg): - arg = _type_check(arg, "Optional[t] requires a single type.") - return Union[arg, type(None)] - - -Optional = _Optional(_root=True) - - -############################################################# -# Subclassable singletons: Generic, Callable, Tuple, Protocol -############################################################# - -class Generic(_FinalTypingBase): - """Abstract base class for generic types. - - A generic type is typically declared by inheriting from - this class parameterized with one or more type variables. - For example, a generic mapping type might be defined as:: - - class Mapping(Generic[KT, VT]): - def __getitem__(self, key: KT) -> VT: - ... - # Etc. - - This class can then be used as follows:: - - def lookup_name(mapping: Mapping[KT, VT], key: KT, default: VT) -> VT: - try: - return mapping[key] - except KeyError: - return default - """ - - __slots__ = () - - def __class_getitem__(self, args): - return GenericType(self, args) - - -class Tuple(_FinalTypingBase): - """Tuple type; Tuple[X, Y] is the cross-product type of X and Y. - - Example: Tuple[T1, T2] is a tuple of two elements corresponding - to type variables T1 and T2. Tuple[int, float, str] is a tuple - of an int, a float and a string. - - To specify a variable-length tuple of homogeneous type, use Tuple[T, ...]. - """ - - __slots__ = () - - def __class_getitem__(self, args): - return GenericType(tuple, args, name='Tuple', call=False) - - -class Callable(_FinalTypingBase): - """Callable type; Callable[[int], str] is a function of (int) -> str. - - The subscription syntax must always be used with exactly two - values: the argument list and the return type. The argument list - must be a list of types or ellipsis; the return type must be a single type. - - There is no syntax to indicate optional or keyword arguments, - such function types are rarely used as callback types. - """ - - __slots__ = () - - def __class_getitem__(self, args): - return GenericType(collections.abc.Callable, args) - - -class Protocol(_FinalTypingBase): - """Internal base class for protocol classes. - - This implements a simple-minded structural issubclass check - (similar but more general than the one-offs in collections.abc - such as Hashable). - """ - - __slots__ = () - - def __class_getitem__(self, args): - return GenericType(self, args) - - -######################################### -# Fancy classes: NamedTuple and TypedDict -######################################### - -def _make_nmtuple(name, types): - msg = "NamedTuple('Name', [(f0, t0), (f1, t1), ...]); each t must be a type" - types = [(n, _type_check(t, msg)) for n, t in types] - nm_tpl = collections.namedtuple(name, [n for n, t in types]) - # Prior to PEP 526, only _field_types attribute was assigned. - # Now, both __annotations__ and _field_types are used to maintain compatibility. - nm_tpl.__annotations__ = nm_tpl._field_types = collections.OrderedDict(types) - try: - nm_tpl.__module__ = sys._getframe(2).f_globals.get('__name__', '__main__') - except (AttributeError, ValueError): - pass - return nm_tpl - - -_PY36 = sys.version_info[:2] >= (3, 6) - -# attributes prohibited to set in NamedTuple class syntax -_prohibited = ('__new__', '__init__', '__slots__', '__getnewargs__', - '_fields', '_field_defaults', '_field_types', - '_make', '_replace', '_asdict', '_source') - -_special = ('__module__', '__name__', '__qualname__', '__annotations__') - - -class NamedTupleMeta(type): - - def __new__(cls, typename, bases, ns): - if ns.get('_root', False): - return super().__new__(cls, typename, bases, ns) - if not _PY36: - raise TypeError("Class syntax for NamedTuple is only supported" - " in Python 3.6+") - types = ns.get('__annotations__', {}) - nm_tpl = _make_nmtuple(typename, types.items()) - defaults = [] - defaults_dict = {} - for field_name in types: - if field_name in ns: - default_value = ns[field_name] - defaults.append(default_value) - defaults_dict[field_name] = default_value - elif defaults: - raise TypeError("Non-default namedtuple field {field_name} cannot " - "follow default field(s) {default_names}" - .format(field_name=field_name, - default_names=', '.join(defaults_dict.keys()))) - nm_tpl.__new__.__defaults__ = tuple(defaults) - nm_tpl._field_defaults = defaults_dict - # update from user namespace without overriding special namedtuple attributes - for key in ns: - if key in _prohibited: - raise AttributeError("Cannot overwrite NamedTuple attribute " + key) - elif key not in _special and key not in nm_tpl._fields: - setattr(nm_tpl, key, ns[key]) - return nm_tpl - - -class NamedTuple(metaclass=NamedTupleMeta): - """Typed version of namedtuple. - - Usage in Python versions >= 3.6:: - - class Employee(NamedTuple): - name: str - id: int - - This is equivalent to:: - - Employee = collections.namedtuple('Employee', ['name', 'id']) - - The resulting class has extra __annotations__ and _field_types - attributes, giving an ordered dict mapping field names to types. - __annotations__ should be preferred, while _field_types - is kept to maintain pre PEP 526 compatibility. (The field names - are in the _fields attribute, which is part of the namedtuple - API.) Alternative equivalent keyword syntax is also accepted:: - - Employee = NamedTuple('Employee', name=str, id=int) - - In Python versions <= 3.5 use:: - - Employee = NamedTuple('Employee', [('name', str), ('id', int)]) - """ - _root = True - - def __new__(self, typename, fields=None, **kwargs): - if kwargs and not _PY36: - raise TypeError("Keyword syntax for NamedTuple is only supported" - " in Python 3.6+") - if fields is None: - fields = kwargs.items() - elif kwargs: - raise TypeError("Either list of fields or keywords" - " can be provided to NamedTuple, not both") - return _make_nmtuple(typename, fields) - - -################################# -# Public functions (with helpers) -################################# - -def cast(typ, val): - """Cast a value to a type. - - This returns the value unchanged. To the type checker this - signals that the return value has the designated type, but at - runtime we intentionally don't check anything (we want this - to be as fast as possible). - """ - return val - - -def _get_defaults(func): - """Internal helper to extract the default arguments, by name.""" - try: - code = func.__code__ - except AttributeError: - # Some built-in functions don't have __code__, __defaults__, etc. - return {} - pos_count = code.co_argcount - arg_names = code.co_varnames - arg_names = arg_names[:pos_count] - defaults = func.__defaults__ or () - kwdefaults = func.__kwdefaults__ - res = dict(kwdefaults) if kwdefaults else {} - pos_offset = pos_count - len(defaults) - for name, value in zip(arg_names[pos_offset:], defaults): - assert name not in res - res[name] = value - return res - - -_allowed_types = (types.FunctionType, types.BuiltinFunctionType, - types.MethodType, types.ModuleType, - WrapperDescriptorType, MethodWrapperType, MethodDescriptorType) - - -def get_type_hints(obj, globalns=None, localns=None): - """Return type hints for an object. - - This is often the same as obj.__annotations__, but it handles - forward references encoded as string literals, and if necessary - adds Optional[t] if a default value equal to None is set. - - The argument may be a module, class, method, or function. The annotations - are returned as a dictionary. For classes, annotations include also - inherited members. - - TypeError is raised if the argument is not of a type that can contain - annotations, and an empty dictionary is returned if no annotations are - present. - - BEWARE -- the behavior of globalns and localns is counterintuitive - (unless you are familiar with how eval() and exec() work). The - search order is locals first, then globals. - - - If no dict arguments are passed, an attempt is made to use the - globals from obj, and these are also used as the locals. If the - object does not appear to have globals, an exception is raised. - - - If one dict argument is passed, it is used for both globals and - locals. - - - If two dict arguments are passed, they specify globals and - locals, respectively. - """ - - if getattr(obj, '__no_type_check__', None): - return {} - if globalns is None: - globalns = getattr(obj, '__globals__', {}) - if localns is None: - localns = globalns - elif localns is None: - localns = globalns - # Classes require a special treatment. - if isinstance(obj, type): - hints = {} - for base in reversed(obj.__mro__): - ann = base.__dict__.get('__annotations__', {}) - for name, value in ann.items(): - if value is None: - value = type(None) - if isinstance(value, str): - value = _ForwardRef(value) - value = _eval_type(value, globalns, localns) - hints[name] = value - return hints - hints = getattr(obj, '__annotations__', None) - if hints is None: - # Return empty annotations for something that _could_ have them. - if isinstance(obj, _allowed_types): - return {} - else: - raise TypeError('{!r} is not a module, class, method, ' - 'or function.'.format(obj)) - defaults = _get_defaults(obj) - hints = dict(hints) - for name, value in hints.items(): - if value is None: - value = type(None) - if isinstance(value, str): - value = _ForwardRef(value) - value = _eval_type(value, globalns, localns) - if name in defaults and defaults[name] is None: - value = Optional[value] - hints[name] = value - return hints - - -def no_type_check(arg): - """Decorator to indicate that annotations are not type hints. - - The argument must be a class or function; if it is a class, it - applies recursively to all methods and classes defined in that class - (but not to methods defined in its superclasses or subclasses). - - This mutates the function(s) or class(es) in place. - """ - if isinstance(arg, type): - arg_attrs = arg.__dict__.copy() - for attr, val in arg.__dict__.items(): - if val in arg.__bases__ + (arg,): - arg_attrs.pop(attr) - for obj in arg_attrs.values(): - if isinstance(obj, types.FunctionType): - obj.__no_type_check__ = True - if isinstance(obj, type): - no_type_check(obj) - try: - arg.__no_type_check__ = True - except TypeError: # built-in classes - pass - return arg - - -def no_type_check_decorator(decorator): - """Decorator to give another decorator the @no_type_check effect. - - This wraps the decorator with something that wraps the decorated - function in @no_type_check. - """ - - @functools.wraps(decorator) - def wrapped_decorator(*args, **kwds): - func = decorator(*args, **kwds) - func = no_type_check(func) - return func - - return wrapped_decorator - - -def _overload_dummy(*args, **kwds): - """Helper for @overload to raise when called.""" - raise NotImplementedError( - "You should not call an overloaded function. " - "A series of @overload-decorated functions " - "outside a stub module should always be followed " - "by an implementation that is not @overload-ed.") - - -def overload(func): - """Decorator for overloaded functions/methods. - - In a stub file, place two or more stub definitions for the same - function in a row, each decorated with @overload. For example: - - @overload - def utf8(value: None) -> None: ... - @overload - def utf8(value: bytes) -> bytes: ... - @overload - def utf8(value: str) -> bytes: ... - - In a non-stub file (i.e. a regular .py file), do the same but - follow it with an implementation. The implementation should *not* - be decorated with @overload. For example: - - @overload - def utf8(value: None) -> None: ... - @overload - def utf8(value: bytes) -> bytes: ... - @overload - def utf8(value: str) -> bytes: ... - def utf8(value): - # implementation goes here - """ - return _overload_dummy - - -def NewType(name, tp): - """NewType creates simple unique types with almost zero - runtime overhead. NewType(name, tp) is considered a subtype of tp - by static type checkers. At runtime, NewType(name, tp) returns - a dummy function that simply returns its argument. Usage:: - - UserId = NewType('UserId', int) - - def name_by_id(user_id: UserId) -> str: - ... - - UserId('user') # Fails type check - - name_by_id(42) # Fails type check - name_by_id(UserId(42)) # OK - - num = UserId(5) + 1 # type: int - """ - - def new_type(x): - return x - - new_type.__name__ = name - new_type.__supertype__ = tp - return new_type - - -###################################################################### -# Various generics mimicking those in collections.abc and -# additional protocols. A few are simply re-exported for completeness. -###################################################################### - -Hashable = collections.abc.Hashable # Not generic. - -Awaitable = GenericType(collections.abc.Awaitable, [T_co]) -Coroutine = GenericType(collections.abc.Coroutine, [T_co, T_contra, V_co]) -AsyncIterable = GenericType(collections.abc.AsyncIterable, [T_co]) -AsyncIterator = GenericType(collections.abc.AsyncIterator, [T_co]) - -Iterable = GenericType(collections.abc.Iterable, [T_co]) -Iterator = GenericType(collections.abc.Iterator, [T_co]) - -class SupportsInt(Protocol): - __slots__ = () - - @abstractmethod - def __int__(self) -> int: - pass - - -class SupportsFloat(Protocol): - __slots__ = () - - @abstractmethod - def __float__(self) -> float: - pass - - -class SupportsComplex(Protocol): - __slots__ = () - - @abstractmethod - def __complex__(self) -> complex: - pass - - -class SupportsBytes(Protocol): - __slots__ = () - - @abstractmethod - def __bytes__(self) -> bytes: - pass - - -class SupportsAbs(Protocol[T_co]): - __slots__ = () - - @abstractmethod - def __abs__(self) -> T_co: - pass - - -class SupportsRound(Protocol[T_co]): - __slots__ = () - - @abstractmethod - def __round__(self, ndigits: int = 0) -> T_co: - pass - - -Reversible = GenericType(collections.abc.Reversible, [T_co]) -Sized = collections.abc.Sized # Not generic. -Container = GenericType(collections.abc.Container, [T_co]) -Collection = GenericType(collections.abc.Collection, [T_co]) - -# Callable was defined earlier. - -AbstractSet = GenericType(collections.abc.Set, [T_co]) -MutableSet = GenericType(collections.abc.MutableSet, [T]) -Set = GenericType(set, [T], name='Set') -FrozenSet = GenericType(frozenset, [T_co], name='FrozenSet') - -# NOTE: It is only covariant in the value type. -Mapping = GenericType(collections.abc.Mapping, [KT, VT_co]) -MutableMapping = GenericType(collections.abc.MutableMapping, [KT, VT]) -Dict = GenericType(dict, [KT, VT], call=False, name='Dict') -DefaultDict = GenericType(collections.defaultdict, [KT, VT], call=False, name='DefaultDict') -Counter = GenericType(collections.Counter, [T], call=False) -ChainMap = GenericType(collections.ChainMap, [KT, VT], call=False) - -Sequence = GenericType(collections.abc.Sequence, [T_co]) -MutableSequence = GenericType(collections.abc.MutableSequence, [T]) -ByteString = collections.abc.ByteString # Not generic. -List = GenericType(list, [T], name='List') -Deque = GenericType(collections.deque, [T], name='Deque') - -MappingView = GenericType(collections.abc.MappingView, [T_co]) -KeysView = GenericType(collections.abc.KeysView, [KT]) -ItemsView = GenericType(collections.abc.ItemsView, [KT, VT_co]) -ValuesView = GenericType(collections.abc.ValuesView, [VT_co]) - -ContextManager = GenericType(contextlib.AbstractContextManager, [T_co]) -if hasattr(contextlib, 'AbstractAsyncContextManager'): - AsyncContextManager = GenericType(contextlib.AbstractAsyncContextManager, [T_co]) - __all__.append('AsyncContextManager') - -Generator = GenericType(collections.abc.Generator, [T_co, T_contra, V_co]) -AsyncGenerator = GenericType(collections.abc.AsyncGenerator, [T_co, T_contra]) - -# Internal type variable used for Type[]. -CT_co = TypeVar('CT_co', covariant=True, bound=type) -# This is not a real generic class. Don't use outside annotations. -Type = GenericType(type, [CT_co], name='Type') -"""A special construct usable to annotate class objects. - -For example, suppose we have the following classes:: - - class User: ... # Abstract base for User classes - class BasicUser(User): ... - class ProUser(User): ... - class TeamUser(User): ... - -And a function that takes a class argument that's a subclass of -User and returns an instance of the corresponding class:: - - U = TypeVar('U', bound=User) - def new_user(user_class: Type[U]) -> U: - user = user_class() - # (Here we could write the user object to a database) - return user - - joe = new_user(BasicUser) - -At this point the type checker knows that joe has type BasicUser. -""" - -################## -# Public constants -################## - -# Python-version-specific alias (Python 2: unicode; Python 3: str) -Text = str - - -# Constant that's True when type checking, but False here. -TYPE_CHECKING = False - -######################################### -# Special io types and re generic aliases -######################################### - - -class IO(Generic[AnyStr]): - """Generic base class for TextIO and BinaryIO. - - This is an abstract, generic version of the return of open(). - - NOTE: This does not distinguish between the different possible - classes (text vs. binary, read vs. write vs. read/write, - append-only, unbuffered). The TextIO and BinaryIO subclasses - below capture the distinctions between text vs. binary, which is - pervasive in the interface; however we currently do not offer a - way to track the other distinctions in the type system. - """ - - __slots__ = () - - @abstractproperty - def mode(self) -> str: - pass - - @abstractproperty - def name(self) -> str: - pass - - @abstractmethod - def close(self) -> None: - pass - - @abstractmethod - def closed(self) -> bool: - pass - - @abstractmethod - def fileno(self) -> int: - pass - - @abstractmethod - def flush(self) -> None: - pass - - @abstractmethod - def isatty(self) -> bool: - pass - - @abstractmethod - def read(self, n: int = -1) -> AnyStr: - pass - - @abstractmethod - def readable(self) -> bool: - pass - - @abstractmethod - def readline(self, limit: int = -1) -> AnyStr: - pass - - @abstractmethod - def readlines(self, hint: int = -1) -> List[AnyStr]: - pass - - @abstractmethod - def seek(self, offset: int, whence: int = 0) -> int: - pass - - @abstractmethod - def seekable(self) -> bool: - pass - - @abstractmethod - def tell(self) -> int: - pass - - @abstractmethod - def truncate(self, size: int = None) -> int: - pass - - @abstractmethod - def writable(self) -> bool: - pass - - @abstractmethod - def write(self, s: AnyStr) -> int: - pass - - @abstractmethod - def writelines(self, lines: List[AnyStr]) -> None: - pass - - @abstractmethod - def __enter__(self) -> 'IO[AnyStr]': - pass - - @abstractmethod - def __exit__(self, type, value, traceback) -> None: - pass - - -class BinaryIO(IO[bytes]): - """Typed version of the return of open() in binary mode.""" - - __slots__ = () - - @abstractmethod - def write(self, s: Union[bytes, bytearray]) -> int: - pass - - @abstractmethod - def __enter__(self) -> 'BinaryIO': - pass - - -class TextIO(IO[str]): - """Typed version of the return of open() in text mode.""" - - __slots__ = () - - @abstractproperty - def buffer(self) -> BinaryIO: - pass - - @abstractproperty - def encoding(self) -> str: - pass - - @abstractproperty - def errors(self) -> Optional[str]: - pass - - @abstractproperty - def line_buffering(self) -> bool: - pass - - @abstractproperty - def newlines(self) -> Any: - pass - - @abstractmethod - def __enter__(self) -> 'TextIO': - pass - - -class io: - """Wrapper namespace for IO generic classes.""" - - __all__ = ['IO', 'TextIO', 'BinaryIO'] - IO = IO - TextIO = TextIO - BinaryIO = BinaryIO - - -io.__name__ = __name__ + '.io' -sys.modules[io.__name__] = io - - -Pattern = GenericType(type(stdlib_re.compile('')), [AnyStr], name='Pattern') -Match = GenericType(type(stdlib_re.match('', '')), [AnyStr], name='Match') - - -class re: - """Wrapper namespace for re type aliases.""" - - __all__ = ['Pattern', 'Match'] - Pattern = Pattern - Match = Match - - -re.__name__ = __name__ + '.re' -sys.modules[re.__name__] = re From a8437fa175c437cede74aadfd3b60cbf0de3fcd6 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 7 Sep 2017 11:29:29 +0200 Subject: [PATCH 11/82] Rename base_subclass to subclass_base --- Python/bltinmodule.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 215a2059b518f3..d710ded2199b45 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -61,7 +61,7 @@ update_bases(PyObject* bases, PyObject** args, int nargs, int* modified_bases) if (PyType_Check(base)) { continue; } - new_base_meth = PyObject_GetAttrString(base, "__base_subclass__"); + new_base_meth = PyObject_GetAttrString(base, "__subclass_base__"); if (!new_base_meth) { if (PyErr_ExceptionMatches(PyExc_AttributeError)) { PyErr_Clear(); @@ -71,7 +71,7 @@ update_bases(PyObject* bases, PyObject** args, int nargs, int* modified_bases) } if (!PyCallable_Check(new_base_meth)) { PyErr_SetString(PyExc_TypeError, - "__base_subclass__ must be callable"); + "__subclass_base__ must be callable"); return NULL; } new_base = _PyObject_FastCall(new_base_meth, stack, 1); From c2d8ac2f8dd8c53ba6c655544f10b33b957305cd Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 7 Sep 2017 12:11:18 +0200 Subject: [PATCH 12/82] Start adding tests --- Lib/test/test_genericclass.py | 42 +++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 Lib/test/test_genericclass.py diff --git a/Lib/test/test_genericclass.py b/Lib/test/test_genericclass.py new file mode 100644 index 00000000000000..da26269bc848ea --- /dev/null +++ b/Lib/test/test_genericclass.py @@ -0,0 +1,42 @@ +import unittest + + +class TestSubclassBase(unittest.TestCase): + def test_subclass_base(self): + pass + + def test_subclass_base_with_builtins(self): + pass + + def test_subclass_base_errors(self): + pass + + def test_subclass_base_wrong(self): + pass + + def test_subclass_base_metaclass(self): + pass + + def test_subclass_base_type_call(self): + pass + + +class TestClassGetitem(unittest.TestCase): + def test_class_getitem(self): + pass + + def test_class_getitem_with_builtins(self): + pass + + def test_class_getitem_errors(self): + pass + + def test_class_getitem_inheritance(self): + pass + + def test_class_getitem_metaclass(self): + pass + + +if __name__ == "__main__": + unittest.main() From bc06c6b3fa22d2ec00c672b386ab808579b691bb Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 8 Sep 2017 11:57:53 +0200 Subject: [PATCH 13/82] Add tests for __subclass_base__ --- Lib/test/test_genericclass.py | 131 ++++++++++++++++++++++++++++++++-- 1 file changed, 124 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_genericclass.py b/Lib/test/test_genericclass.py index da26269bc848ea..020633fe37f8b8 100644 --- a/Lib/test/test_genericclass.py +++ b/Lib/test/test_genericclass.py @@ -2,23 +2,140 @@ class TestSubclassBase(unittest.TestCase): + def test_subclass_base_signature(self): + tested = [] + class B: ... + class C: + def __subclass_base__(self, *args, **kwargs): + tested.extend([args, kwargs]) + return C + c = C() + self.assertEqual(tested, []) + class D(B, c): ... + self.assertEqual(tested[0], ((B, c),)) + self.assertEqual(tested[1], {}) + def test_subclass_base(self): - pass + tested = [] + class A: ... + class B: ... + class C: + def __subclass_base__(self, bases): + tested.append(bases) + return self.__class__ + c = C() + self.assertEqual(tested, []) + class D(A, c, B): ... + self.assertEqual(tested[-1], (A, c, B)) + self.assertEqual(D.__bases__, (A, C, B)) + self.assertEqual(D.__orig_bases__, (A, c, B)) + self.assertEqual(D.__mro__, (D, A, C, B, object)) + d = D() + class E(d): ... + self.assertEqual(tested[-1], (d,)) + self.assertEqual(E.__bases__, (D,)) + + def test_subclass_base_none(self): + tested = [] + class A: ... + class B: ... + class C: + def __subclass_base__(self, bases): + tested.append(bases) + return None + c = C() + self.assertEqual(tested, []) + class D(A, c, B): ... + self.assertEqual(tested[-1], (A, c, B)) + self.assertEqual(D.__bases__, (A, B)) + self.assertEqual(D.__orig_bases__, (A, c, B)) + self.assertEqual(D.__mro__, (D, A, B, object)) + class E(c): ... + self.assertEqual(tested[-1], (c,)) + self.assertEqual(E.__bases__, (object,)) + self.assertEqual(E.__orig_bases__, (c,)) + self.assertEqual(E.__mro__, (E, object)) def test_subclass_base_with_builtins(self): - pass + tested = [] + class A: ... + class C: + def __subclass_base__(self, bases): + tested.append(bases) + return dict + c = C() + self.assertEqual(tested, []) + class D(A, c): ... + self.assertEqual(tested[-1], (A, c)) + self.assertEqual(D.__bases__, (A, dict)) + self.assertEqual(D.__orig_bases__, (A, c)) + self.assertEqual(D.__mro__, (D, A, dict, object)) + + def test_subclass_base_with_builtins_2(self): + tested = [] + class C: + def __subclass_base__(self, bases): + tested.append(bases) + return C + c = C() + self.assertEqual(tested, []) + class D(c, dict): ... + self.assertEqual(tested[-1], (c, dict)) + self.assertEqual(D.__bases__, (C, dict)) + self.assertEqual(D.__orig_bases__, (c, dict)) + self.assertEqual(D.__mro__, (D, C, dict, object)) def test_subclass_base_errors(self): - pass + class C_too_many: + def __subclass_base__(self, bases, something, other): + return None + c = C_too_many() + with self.assertRaises(TypeError): + class D(c): ... + class C_too_few: + def __subclass_base__(self, bases, something, other): + return None + d = C_too_few() + with self.assertRaises(TypeError): + class D(d): ... - def test_subclass_base_wrong(self): - pass + def test_subclass_base_errors_2(self): + class C_not_callable: + __subclass_base__ = "Surprise!" + c = C_not_callable() + with self.assertRaises(TypeError): + class D(c): ... def test_subclass_base_metaclass(self): - pass + meta_args = [] + class Meta(type): + def __new__(mcls, name, bases, ns): + meta_args.extend([mcls, name, bases, ns]) + return super().__new__(mcls, name, bases, ns) + class A: ... + class C: + def __subclass_base__(self, bases): + return A + c = C() + class D(c, metaclass=Meta): + x = 1 + self.assertEqual(meta_args[0], Meta) + self.assertEqual(meta_args[1], 'D') + self.assertEqual(meta_args[2], (A,)) + self.assertEqual(meta_args[3]['x'], 1) + self.assertEqual(D.__bases__, (A,)) + self.assertEqual(D.__orig_bases__, (c,)) + self.assertEqual(D.__mro__, (D, A, object)) + self.assertEqual(D.__class__, Meta) def test_subclass_base_type_call(self): - pass + # Substitution should _not_ happen in direct type call + class C: + def __subclass_base__(self, bases): + return None + c = C() + with self.assertRaises(TypeError): + type('Bad', (c,), {}) class TestClassGetitem(unittest.TestCase): From 86c5d615fd1c1660a1325300f3e0199c6a276225 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 8 Sep 2017 15:21:06 +0200 Subject: [PATCH 14/82] Add tests for __class_getitem__ --- Lib/test/test_genericclass.py | 103 +++++++++++++++++++++++++++++++--- 1 file changed, 94 insertions(+), 9 deletions(-) diff --git a/Lib/test/test_genericclass.py b/Lib/test/test_genericclass.py index 020633fe37f8b8..76ec8000aa5b65 100644 --- a/Lib/test/test_genericclass.py +++ b/Lib/test/test_genericclass.py @@ -140,19 +140,104 @@ def __subclass_base__(self, bases): class TestClassGetitem(unittest.TestCase): def test_class_getitem(self): - pass - + getitem_args = [] + class C: + def __class_getitem__(*args, **kwargs): + getitem_args.extend([args, kwargs]) + return None + C[int, str] + self.assertEqual(getitem_args[0], (C, (int, str))) + self.assertEqual(getitem_args[1], {}) + + def test_class_getitem(self): + class C: + def __class_getitem__(cls, item): + return f'C[{item.__name__}]' + self.assertEqual(C[int], 'C[int]') + self.assertEqual(C[C], 'C[C]') + + def test_class_getitem_inheritance(self): + class C: + def __class_getitem__(cls, item): + return f'{cls.__name__}[{item.__name__}]' + class D(C): ... + self.assertEqual(D[int], 'D[int]') + self.assertEqual(D[D], 'D[D]') + + def test_class_getitem_inheritance_2(self): + class C: + def __class_getitem__(cls, item): + return 'Should not see this' + class D(C): + def __class_getitem__(cls, item): + return f'{cls.__name__}[{item.__name__}]' + self.assertEqual(D[int], 'D[int]') + self.assertEqual(D[D], 'D[D]') + + def test_class_getitem_patched(self): + class C: + def __init_subclass__(cls): + def __class_getitem__(cls, item): + return f'{cls.__name__}[{item.__name__}]' + cls.__class_getitem__ = __class_getitem__ + class D(C): ... + self.assertEqual(D[int], 'D[int]') + self.assertEqual(D[D], 'D[D]') + def test_class_getitem_with_builtins(self): - pass + class A(dict): + called_with = None + + def __class_getitem__(cls, item): + cls.called_with = item + class B(A): + pass + self.assertIs(B.called_with, None) + B[int] + self.assertIs(B.called_with, int) def test_class_getitem_errors(self): - pass - - def test_class_getitem_inheritance(self): - pass - + class C_too_few: + def __class_getitem__(cls): + return None + with self.assertRaises(TypeError): + C_too_few[int] + class C_too_many: + def __class_getitem__(cls, one, two): + return None + with self.assertRaises(TypeError): + C_too_many[int] + + def test_class_getitem_errors_2(self): + class C: + def __class_getitem__(cls, item): + return None + with self.assertRaises(TypeError): + C()[int] + class E: ... + e = E() + e.__class_getitem__ = lambda cls, item: 'This will not work' + with self.assertRaises(TypeError): + e[int] + class C_not_callable: + __class_getitem__ = "Surprise!" + with self.assertRaises(TypeError): + C_not_callable[int] + def test_class_getitem_metaclass(self): - pass + class Meta(type): + def __class_getitem__(cls, item): + return f'{cls.__name__}[{item.__name__}]' + self.assertEqual(Meta[int], 'Meta[int]') + + def test_class_getitem_metaclass_2(self): + class Meta(type): + def __getitem__(cls, item): + return 'from metaclass' + class C(metaclass=Meta): + def __class_getitem__(cls, item): + return 'from __class_getitem__' + self.assertEqual(C[int], 'from metaclass') if __name__ == "__main__": From 2cef78ae92353ca667730d2a1d7a81f36f9501a9 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 9 Sep 2017 22:38:44 +0200 Subject: [PATCH 15/82] Fix trailing whitespace --- Lib/test/test_genericclass.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_genericclass.py b/Lib/test/test_genericclass.py index 76ec8000aa5b65..a61b9e9ee9c3c4 100644 --- a/Lib/test/test_genericclass.py +++ b/Lib/test/test_genericclass.py @@ -195,7 +195,7 @@ class B(A): self.assertIs(B.called_with, None) B[int] self.assertIs(B.called_with, int) - + def test_class_getitem_errors(self): class C_too_few: def __class_getitem__(cls): From b9bbf7c569027cc9bd16d731683a08ea3838ecde Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 10 Sep 2017 17:58:11 +0200 Subject: [PATCH 16/82] Alternative implementation of __class_getitem__ --- Objects/abstract.c | 17 +++++++++++++++++ Objects/typeobject.c | 32 +------------------------------- 2 files changed, 18 insertions(+), 31 deletions(-) diff --git a/Objects/abstract.c b/Objects/abstract.c index 3cb7a32b01ee5a..0d75067d36bf33 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -143,6 +143,7 @@ PyObject * PyObject_GetItem(PyObject *o, PyObject *key) { PyMappingMethods *m; + PyObject *meth, *stack[2] = {o, key}; if (o == NULL || key == NULL) { return null_error(); @@ -168,6 +169,22 @@ PyObject_GetItem(PyObject *o, PyObject *key) "be integer, not '%.200s'", key); } + if (PyType_Check(o)) { + meth = PyObject_GetAttrString(o, "__class_getitem__"); + if (meth) { + if (!PyCallable_Check(meth)) { + PyErr_SetString(PyExc_TypeError, + "__class_getitem__ must be callable"); + return NULL; + } + return _PyObject_FastCall(meth, stack, 2); + } + else if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { + return NULL; + } + PyErr_Clear(); + } + return type_error("'%.200s' object is not subscriptable", o); } diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 66fb921c3c615e..2a8118b43c5a0a 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -3488,36 +3488,6 @@ type_is_gc(PyTypeObject *type) return type->tp_flags & Py_TPFLAGS_HEAPTYPE; } -static PyObject * -generic_subscript(PyTypeObject *tp, PyObject *key) -{ - PyObject* stack[2] = {(PyObject *)tp, key}; - const char* msg = "'%.200s' object is not subscriptable"; - PyObject *meth = PyObject_GetAttrString((PyObject *)tp, "__class_getitem__"); - if (meth) { - if (!PyCallable_Check(meth)) { - PyErr_SetString(PyExc_TypeError, - "__class_getitem__ must be callable"); - return NULL; - } - return _PyObject_FastCall(meth, stack, 2); - } - else { - if (PyErr_ExceptionMatches(PyExc_AttributeError)) { - PyErr_Clear(); - PyErr_Format(PyExc_TypeError, msg, ((PyObject *)tp)->ob_type->tp_name); - return NULL; - } - return NULL; - } -} - -static PyMappingMethods generic_as_mapping = { - NULL, /*mp_length*/ - (binaryfunc)generic_subscript, /*mp_subscript*/ - NULL, /*mp_ass_subscript*/ -}; - PyTypeObject PyType_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "type", /* tp_name */ @@ -3531,7 +3501,7 @@ PyTypeObject PyType_Type = { (reprfunc)type_repr, /* tp_repr */ 0, /* tp_as_number */ 0, /* tp_as_sequence */ - &generic_as_mapping, /* tp_as_mapping */ + 0, /* tp_as_mapping */ 0, /* tp_hash */ (ternaryfunc)type_call, /* tp_call */ 0, /* tp_str */ From 2495e942c7f90ff7b2292ef44ec5d82bf2281619 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 10 Sep 2017 19:36:32 +0200 Subject: [PATCH 17/82] Simplify code and fix reference counting --- Objects/abstract.c | 7 +++++-- Python/bltinmodule.c | 49 ++++++++++++++++++++++---------------------- 2 files changed, 29 insertions(+), 27 deletions(-) diff --git a/Objects/abstract.c b/Objects/abstract.c index 0d75067d36bf33..ec11a1e5f3f789 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -143,7 +143,7 @@ PyObject * PyObject_GetItem(PyObject *o, PyObject *key) { PyMappingMethods *m; - PyObject *meth, *stack[2] = {o, key}; + PyObject *meth, *result, *stack[2] = {o, key}; if (o == NULL || key == NULL) { return null_error(); @@ -175,9 +175,12 @@ PyObject_GetItem(PyObject *o, PyObject *key) if (!PyCallable_Check(meth)) { PyErr_SetString(PyExc_TypeError, "__class_getitem__ must be callable"); + Py_DECREF(meth); return NULL; } - return _PyObject_FastCall(meth, stack, 2); + result = _PyObject_FastCall(meth, stack, 2); + Py_DECREF(meth); + return result; } else if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { return NULL; diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index d710ded2199b45..0341aee155311c 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -49,9 +49,9 @@ _Py_IDENTIFIER(stderr); static PyObject* update_bases(PyObject* bases, PyObject** args, int nargs, int* modified_bases) { - int i, ind, tot_nones; - PyObject *base, *new_base, *new_base_meth, *new_bases; - PyObject* stack[1] = {bases}; + int i, ind, tot_nones = 0; + PyObject *base, *meth, *new_base, *new_bases; + PyObject *stack[1] = {bases}; assert(PyTuple_Check(bases)); /* We have a separate cycle to calculate replacements with the idea that in @@ -61,45 +61,41 @@ update_bases(PyObject* bases, PyObject** args, int nargs, int* modified_bases) if (PyType_Check(base)) { continue; } - new_base_meth = PyObject_GetAttrString(base, "__subclass_base__"); - if (!new_base_meth) { - if (PyErr_ExceptionMatches(PyExc_AttributeError)) { - PyErr_Clear(); - continue; + if (!(meth = PyObject_GetAttrString(base, "__subclass_base__"))) { + if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { + return NULL; } - return NULL; + PyErr_Clear(); + continue; } - if (!PyCallable_Check(new_base_meth)) { + if (!PyCallable_Check(meth)) { PyErr_SetString(PyExc_TypeError, "__subclass_base__ must be callable"); + Py_DECREF(meth); return NULL; } - new_base = _PyObject_FastCall(new_base_meth, stack, 1); - if (!new_base){ + if (!(new_base = _PyObject_FastCall(meth, stack, 1))){ + Py_DECREF(meth); return NULL; } - Py_INCREF(new_base); + if (new_base == Py_None) { + tot_nones++; + } + Py_DECREF(base); args[i] = new_base; *modified_bases = 1; + Py_DECREF(meth); } if (!*modified_bases){ return bases; } - /* Find out have many bases wants to be removed to pre-allocate - the tuple for new bases, then keep only non-None's in bases.*/ - tot_nones = 0; - for (i = 2; i < nargs; i++) { - if (args[i] == Py_None) { - tot_nones++; - } - } new_bases = PyTuple_New(nargs - 2 - tot_nones); ind = 0; for (i = 2; i < nargs; i++) { - base = args[i]; - if (base != Py_None) { - Py_INCREF(base); - PyTuple_SET_ITEM(new_bases, ind, base); + new_base = args[i]; + if (new_base != Py_None) { + Py_INCREF(new_base); + PyTuple_SET_ITEM(new_bases, ind, new_base); ind++; } } @@ -281,6 +277,9 @@ builtin___build_class__(PyObject *self, PyObject **args, Py_ssize_t nargs, Py_DECREF(meta); Py_XDECREF(mkw); Py_DECREF(bases); + if (modified_bases) { + Py_DECREF(old_bases); + } return cls; } From 9ca8dfca2357d5f1556966c23f346b6d609b98c1 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 11 Nov 2017 01:03:10 +0100 Subject: [PATCH 18/82] Rename __base_subclass__ to __mro_entry__ --- Lib/test/test_genericclass.py | 40 +++++++++++++++++------------------ Python/bltinmodule.c | 4 ++-- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/Lib/test/test_genericclass.py b/Lib/test/test_genericclass.py index a61b9e9ee9c3c4..4e01a75f0d2a4e 100644 --- a/Lib/test/test_genericclass.py +++ b/Lib/test/test_genericclass.py @@ -1,12 +1,12 @@ import unittest -class TestSubclassBase(unittest.TestCase): - def test_subclass_base_signature(self): +class TestMROEntry(unittest.TestCase): + def test_mro_entry_signature(self): tested = [] class B: ... class C: - def __subclass_base__(self, *args, **kwargs): + def __mro_entry__(self, *args, **kwargs): tested.extend([args, kwargs]) return C c = C() @@ -15,12 +15,12 @@ class D(B, c): ... self.assertEqual(tested[0], ((B, c),)) self.assertEqual(tested[1], {}) - def test_subclass_base(self): + def test_mro_entry(self): tested = [] class A: ... class B: ... class C: - def __subclass_base__(self, bases): + def __mro_entry__(self, bases): tested.append(bases) return self.__class__ c = C() @@ -35,12 +35,12 @@ class E(d): ... self.assertEqual(tested[-1], (d,)) self.assertEqual(E.__bases__, (D,)) - def test_subclass_base_none(self): + def test_mro_entry_none(self): tested = [] class A: ... class B: ... class C: - def __subclass_base__(self, bases): + def __mro_entry__(self, bases): tested.append(bases) return None c = C() @@ -56,11 +56,11 @@ class E(c): ... self.assertEqual(E.__orig_bases__, (c,)) self.assertEqual(E.__mro__, (E, object)) - def test_subclass_base_with_builtins(self): + def test_mro_entry_with_builtins(self): tested = [] class A: ... class C: - def __subclass_base__(self, bases): + def __mro_entry__(self, bases): tested.append(bases) return dict c = C() @@ -71,10 +71,10 @@ class D(A, c): ... self.assertEqual(D.__orig_bases__, (A, c)) self.assertEqual(D.__mro__, (D, A, dict, object)) - def test_subclass_base_with_builtins_2(self): + def test_mro_entry_with_builtins_2(self): tested = [] class C: - def __subclass_base__(self, bases): + def __mro_entry__(self, bases): tested.append(bases) return C c = C() @@ -85,28 +85,28 @@ class D(c, dict): ... self.assertEqual(D.__orig_bases__, (c, dict)) self.assertEqual(D.__mro__, (D, C, dict, object)) - def test_subclass_base_errors(self): + def test_mro_entry_errors(self): class C_too_many: - def __subclass_base__(self, bases, something, other): + def __mro_entry__(self, bases, something, other): return None c = C_too_many() with self.assertRaises(TypeError): class D(c): ... class C_too_few: - def __subclass_base__(self, bases, something, other): + def __mro_entry__(self, bases, something, other): return None d = C_too_few() with self.assertRaises(TypeError): class D(d): ... - def test_subclass_base_errors_2(self): + def test_mro_entry_errors_2(self): class C_not_callable: - __subclass_base__ = "Surprise!" + __mro_entry__ = "Surprise!" c = C_not_callable() with self.assertRaises(TypeError): class D(c): ... - def test_subclass_base_metaclass(self): + def test_mro_entry_metaclass(self): meta_args = [] class Meta(type): def __new__(mcls, name, bases, ns): @@ -114,7 +114,7 @@ def __new__(mcls, name, bases, ns): return super().__new__(mcls, name, bases, ns) class A: ... class C: - def __subclass_base__(self, bases): + def __mro_entry__(self, bases): return A c = C() class D(c, metaclass=Meta): @@ -128,10 +128,10 @@ class D(c, metaclass=Meta): self.assertEqual(D.__mro__, (D, A, object)) self.assertEqual(D.__class__, Meta) - def test_subclass_base_type_call(self): + def test_mro_entry_type_call(self): # Substitution should _not_ happen in direct type call class C: - def __subclass_base__(self, bases): + def __mro_entry__(self, bases): return None c = C() with self.assertRaises(TypeError): diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 0341aee155311c..26c112b0d999e5 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -61,7 +61,7 @@ update_bases(PyObject* bases, PyObject** args, int nargs, int* modified_bases) if (PyType_Check(base)) { continue; } - if (!(meth = PyObject_GetAttrString(base, "__subclass_base__"))) { + if (!(meth = PyObject_GetAttrString(base, "__mro_entry__"))) { if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { return NULL; } @@ -70,7 +70,7 @@ update_bases(PyObject* bases, PyObject** args, int nargs, int* modified_bases) } if (!PyCallable_Check(meth)) { PyErr_SetString(PyExc_TypeError, - "__subclass_base__ must be callable"); + "__mro_entry__ must be callable"); Py_DECREF(meth); return NULL; } From 74bd36f8a4cbcb4bc822a56dab701c58fe77f2ec Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 11 Nov 2017 01:27:18 +0100 Subject: [PATCH 19/82] Add types.resolve_bases --- Lib/test/test_types.py | 26 ++++++++++++++++++++++++++ Lib/types.py | 24 ++++++++++++++++++++++-- 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index 28133a3560f3c5..da82daa26a9200 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -844,6 +844,17 @@ def func(ns): self.assertEqual(C.y, 1) self.assertEqual(C.z, 2) + def test_new_class_with_mro_entry(self): + class A: pass + class C: + def __mro_entry__(self, bases): + return A + c = C() + D = types.new_class('D', (c,), {}) + self.assertEqual(D.__bases__, (A,)) + self.assertEqual(D.__orig_bases__, (c,)) + self.assertEqual(D.__mro__, (D, A, object)) + # Many of the following tests are derived from test_descr.py def test_prepare_class(self): # Basic test of metaclass derivation @@ -886,6 +897,21 @@ def __prepare__(*args): class Bar(metaclass=BadMeta()): pass + def test_resolve_bases(self): + class A: pass + class C: + def __mro_entry__(self, bases): + if A in bases: + return None + return A + c = C() + self.assertEqual(types.resolve_bases(()), ()) + self.assertEqual(types.resolve_bases((c,)), (A,)) + self.assertEqual(types.resolve_bases((C,)), (C,)) + self.assertEqual(types.resolve_bases((A, C)), (A, C)) + self.assertEqual(types.resolve_bases((c, A)), (A,)) + self.assertEqual(types.resolve_bases((A, c)), (A,)) + def test_metaclass_derivation(self): # issue1294232: correct metaclass calculation new_calls = [] # to check the order of __new__ calls diff --git a/Lib/types.py b/Lib/types.py index 336918fea09d4a..40e58846a78efa 100644 --- a/Lib/types.py +++ b/Lib/types.py @@ -60,10 +60,30 @@ def _m(self): pass # Provide a PEP 3115 compliant mechanism for class creation def new_class(name, bases=(), kwds=None, exec_body=None): """Create a class object dynamically using the appropriate metaclass.""" - meta, ns, kwds = prepare_class(name, bases, kwds) + resolved_bases = resolve_bases(bases) + meta, ns, kwds = prepare_class(name, resolved_bases, kwds) if exec_body is not None: exec_body(ns) - return meta(name, bases, ns, **kwds) + cls = meta(name, resolved_bases, ns, **kwds) + if resolved_bases is not bases: + cls.__orig_bases__ = bases + return cls + +def resolve_bases(bases): + """Resolve MRO entries dynamically as specified by PEP 560.""" + new_bases = list(bases) + updated = False + for i, base in enumerate(bases): + if isinstance(base, type): + continue + if not hasattr(base, "__mro_entry__"): + continue + new_base = base.__mro_entry__(bases) + updated = True + new_bases[i] = new_base + if not updated: + return bases + return tuple(b for b in new_bases if b is not None) def prepare_class(name, bases=(), kwds=None): """Call the __prepare__ method of the appropriate metaclass. From 6f20c4518bc4220f8f71ca9725e084ddbf976d2b Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 11 Nov 2017 01:31:18 +0100 Subject: [PATCH 20/82] Make Python and C versions as similar as possible --- Lib/types.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Lib/types.py b/Lib/types.py index 40e58846a78efa..c1ed6bdc8b8066 100644 --- a/Lib/types.py +++ b/Lib/types.py @@ -64,10 +64,9 @@ def new_class(name, bases=(), kwds=None, exec_body=None): meta, ns, kwds = prepare_class(name, resolved_bases, kwds) if exec_body is not None: exec_body(ns) - cls = meta(name, resolved_bases, ns, **kwds) if resolved_bases is not bases: - cls.__orig_bases__ = bases - return cls + ns['__orig_bases__'] = bases + return meta(name, resolved_bases, ns, **kwds) def resolve_bases(bases): """Resolve MRO entries dynamically as specified by PEP 560.""" From bcfbcf60fc9db8cbf09e1c308aa64bc8126be727 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 11 Nov 2017 10:13:50 +0100 Subject: [PATCH 21/82] More tests --- Lib/test/test_types.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index da82daa26a9200..f77a8bb363118e 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -855,6 +855,18 @@ def __mro_entry__(self, bases): self.assertEqual(D.__orig_bases__, (c,)) self.assertEqual(D.__mro__, (D, A, object)) + def test_new_class_with_mro_entry_none(self): + class A: pass + class B: pass + class C: + def __mro_entry__(self, bases): + return None + c = C() + D = types.new_class('D', (A, c, B), {}) + self.assertEqual(D.__bases__, (A, B)) + self.assertEqual(D.__orig_bases__, (A, c, B)) + self.assertEqual(D.__mro__, (D, A, B, object)) + # Many of the following tests are derived from test_descr.py def test_prepare_class(self): # Basic test of metaclass derivation @@ -899,6 +911,7 @@ class Bar(metaclass=BadMeta()): def test_resolve_bases(self): class A: pass + class B: pass class C: def __mro_entry__(self, bases): if A in bases: @@ -911,6 +924,12 @@ def __mro_entry__(self, bases): self.assertEqual(types.resolve_bases((A, C)), (A, C)) self.assertEqual(types.resolve_bases((c, A)), (A,)) self.assertEqual(types.resolve_bases((A, c)), (A,)) + x = (A,) + y = (C,) + z = (A, C) + t = (A, C, B) + for bases in [x, y, z, t]: + self.assertIs(types.resolve_bases(bases), bases) def test_metaclass_derivation(self): # issue1294232: correct metaclass calculation From d7bd63080d25ca50fcd1101839dea0202ad7d0c2 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 11 Nov 2017 10:51:06 +0100 Subject: [PATCH 22/82] Fail fast in type.__new__ --- Objects/typeobject.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 2a8118b43c5a0a..8d48bd845df5f8 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -2377,6 +2377,15 @@ type_new(PyTypeObject *metatype, PyObject *args, PyObject *kwds) nbases = 1; } else { + for (i = 0; i < nbases; i++) { + tmp = PyTuple_GET_ITEM(bases, i); + if (!PyType_Check(tmp) && PyObject_GetAttrString(tmp, "__mro_entry__")) { + PyErr_SetString(PyExc_TypeError, + "type() doesn't support MRO entry resolution; " + "use types.new_class()"); + return NULL; + } + } /* Search the bases for the proper metatype to deal with this: */ winner = _PyType_CalculateMetaclass(metatype, bases); if (winner == NULL) { From 29e54c334dc00427644db08fdad2a73bcb2dc8bd Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 11 Nov 2017 10:57:52 +0100 Subject: [PATCH 23/82] Fix a test --- Lib/test/test_genericclass.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_genericclass.py b/Lib/test/test_genericclass.py index 4e01a75f0d2a4e..691ee0b6f775d7 100644 --- a/Lib/test/test_genericclass.py +++ b/Lib/test/test_genericclass.py @@ -93,7 +93,7 @@ def __mro_entry__(self, bases, something, other): with self.assertRaises(TypeError): class D(c): ... class C_too_few: - def __mro_entry__(self, bases, something, other): + def __mro_entry__(self): return None d = C_too_few() with self.assertRaises(TypeError): From cba2a58f8417fa44993c5a5f6113b6476875cb4f Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 11 Nov 2017 11:12:36 +0100 Subject: [PATCH 24/82] Test error message --- Lib/test/test_genericclass.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_genericclass.py b/Lib/test/test_genericclass.py index 691ee0b6f775d7..fde788008cc664 100644 --- a/Lib/test/test_genericclass.py +++ b/Lib/test/test_genericclass.py @@ -134,7 +134,9 @@ class C: def __mro_entry__(self, bases): return None c = C() - with self.assertRaises(TypeError): + with self.assertRaisesRegex(TypeError, + "MRO entry resolution; " + "use types.new_class()"): type('Bad', (c,), {}) From 966a9ea754e38b34a97b55a7e4b7b46d75cccbcd Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 11 Nov 2017 11:12:48 +0100 Subject: [PATCH 25/82] Test error message --- Lib/test/test_genericclass.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_genericclass.py b/Lib/test/test_genericclass.py index fde788008cc664..556c2d2f4fe9ed 100644 --- a/Lib/test/test_genericclass.py +++ b/Lib/test/test_genericclass.py @@ -136,7 +136,7 @@ def __mro_entry__(self, bases): c = C() with self.assertRaisesRegex(TypeError, "MRO entry resolution; " - "use types.new_class()"): + "use type.new_class()"): type('Bad', (c,), {}) From 734c7e9a79796c6f2f685bde1de188ff0b439446 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 11 Nov 2017 11:13:13 +0100 Subject: [PATCH 26/82] Test error message --- Lib/test/test_genericclass.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_genericclass.py b/Lib/test/test_genericclass.py index 556c2d2f4fe9ed..fde788008cc664 100644 --- a/Lib/test/test_genericclass.py +++ b/Lib/test/test_genericclass.py @@ -136,7 +136,7 @@ def __mro_entry__(self, bases): c = C() with self.assertRaisesRegex(TypeError, "MRO entry resolution; " - "use type.new_class()"): + "use types.new_class()"): type('Bad', (c,), {}) From 042760b8956f4c689a1662381afd4dab1df5a068 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 11 Nov 2017 15:24:02 +0100 Subject: [PATCH 27/82] First sketches of typing refactoring --- Lib/typing.py | 1531 +++++++++++++------------------------------------ 1 file changed, 385 insertions(+), 1146 deletions(-) diff --git a/Lib/typing.py b/Lib/typing.py index c00a3a10e1fae1..1168f3c555cd04 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -6,19 +6,8 @@ import re as stdlib_re # Avoid confusion with the re we export. import sys import types -try: - import collections.abc as collections_abc -except ImportError: - import collections as collections_abc # Fallback for PY3.2. -if sys.version_info[:2] >= (3, 6): - import _collections_abc # Needed for private function _check_methods # noqa -try: - from types import WrapperDescriptorType, MethodWrapperType, MethodDescriptorType -except ImportError: - WrapperDescriptorType = type(object.__init__) - MethodWrapperType = type(object().__str__) - MethodDescriptorType = type(str.join) - +import collections.abc as collections_abc +from types import WrapperDescriptorType, MethodWrapperType, MethodDescriptorType # Please keep __all__ alphabetized within each category. __all__ = [ @@ -53,15 +42,13 @@ 'Sequence', 'Sized', 'ValuesView', - # The following are added depending on presence - # of their non-generic counterparts in stdlib: - # Awaitable, - # AsyncIterator, - # AsyncIterable, - # Coroutine, - # Collection, - # AsyncGenerator, - # AsyncContextManager + 'Awaitable', + 'AsyncIterator', + 'AsyncIterable', + 'Coroutine', + 'Collection', + 'AsyncGenerator', + 'AsyncContextManager', # Structural checks, a.k.a. protocols. 'Reversible', @@ -99,72 +86,166 @@ # namespace, but excluded from __all__ because they might stomp on # legitimate imports of those modules. - -def _qualname(x): - if sys.version_info[:2] >= (3, 3): - return x.__qualname__ - else: - # Fall back to just name. - return x.__name__ - +# +# Internal helper functions. +# def _trim_name(nm): - whitelist = ('_TypeAlias', '_ForwardRef', '_TypingBase', '_FinalTypingBase') + whitelist = ('_TypingBase', '_FinalTypingBase', '_SingletonTypingBase', '_ForwardRef') if nm.startswith('_') and nm not in whitelist: nm = nm[1:] return nm -class TypingMeta(type): - """Metaclass for most types defined in typing module - (not a part of public API). +def _get_type_vars(types, tvars): + for t in types: + if isinstance(t, type) and issubclass(t, _TypingBase) or isinstance(t, _TypingBase): + t._get_type_vars(tvars) - This overrides __new__() to require an extra keyword parameter - '_root', which serves as a guard against naive subclassing of the - typing classes. Any legitimate class defined using a metaclass - derived from TypingMeta must pass _root=True. - This also defines a dummy constructor (all the work for most typing - constructs is done in __new__) and a nicer repr(). +def _type_vars(types): + tvars = [] + _get_type_vars(types, tvars) + return tuple(tvars) + + +def _eval_type(t, globalns, localns): + if isinstance(t, type) and issubclass(t, _TypingBase) or isinstance(t, _TypingBase): + return t._eval_type(globalns, localns) + return t + + +Generic = object() +_Protocol = object() + +def _type_check(arg, msg): + """Check that the argument is a type, and return it (internal helper). + + As a special case, accept None and return type(None) instead. + Also, _TypeAlias instances (e.g. Match, Pattern) are acceptable. + + The msg argument is a human-readable error message, e.g. + + "Union[arg, ...]: arg should be a type." + + We append the repr() of the actual value (truncated to 100 chars). """ + if arg is None: + return type(None) + if isinstance(arg, str): + arg = _ForwardRef(arg) + if ( + isinstance(arg, _TypingBase) and type(arg).__name__ == '_ClassVar' or + not isinstance(arg, (type, _TypingBase)) and not callable(arg) + ): + raise TypeError(msg + " Got %.100r." % (arg,)) + # Bare Union etc. are not valid as type arguments + if ( + type(arg).__name__ in ('_Union', '_Optional') and + not getattr(arg, '__origin__', None) or arg in (Generic, _Protocol) + ): + raise TypeError("Plain %s is not valid as type argument" % arg) + return arg - _is_protocol = False - def __new__(cls, name, bases, namespace, *, _root=False): - if not _root: - raise TypeError("Cannot subclass %s" % - (', '.join(map(_type_repr, bases)) or '()')) - return super().__new__(cls, name, bases, namespace) +def _type_repr(obj): + """Return the repr() of an object, special-casing types (internal helper). - def __init__(self, *args, **kwds): - pass + If obj is a type, we return a shorter version than the default + type.__repr__, based on the module and qualified name, which is + typically enough to uniquely identify a type. For everything + else, we fall back on repr(obj). + """ + if isinstance(obj, type) and not issubclass(obj, _TypingBase): + if obj.__module__ == 'builtins': + return obj.__qualname__ + return '%s.%s' % (obj.__module__, obj.__qualname__) + if obj is ...: + return('...') + if isinstance(obj, types.FunctionType): + return obj.__name__ + return repr(obj) - def _eval_type(self, globalns, localns): - """Override this in subclasses to interpret forward references. - For example, List['C'] is internally stored as - List[_ForwardRef('C')], which should evaluate to List[C], - where C is an object found in globalns or localns (searching - localns first, of course). - """ - return self +def _remove_dups_flatten(parameters): + """An internal helper for Union creation and substitution: flatten Union's + among parameters, then remove duplicates and strict subclasses. + """ - def _get_type_vars(self, tvars): - pass + # Flatten out Union[Union[...], ...]. + params = [] + for p in parameters: + if isinstance(p, _GenericAlias) and p.__origin__ is Union: + params.extend(p.__args__) + elif isinstance(p, tuple) and len(p) > 0 and p[0] is Union: + params.extend(p[1:]) + else: + params.append(p) + # Weed out strict duplicates, preserving the first of each occurrence. + all_params = set(params) + if len(all_params) < len(params): + new_params = [] + for t in params: + if t in all_params: + new_params.append(t) + all_params.remove(t) + params = new_params + assert not all_params, all_params + # Weed out subclasses. + # E.g. Union[int, Employee, Manager] == Union[int, Employee]. + # If object is present it will be sole survivor among proper classes. + # Never discard type variables. + # (In particular, Union[str, AnyStr] != AnyStr.) + all_params = set(params) + for t1 in params: + if not isinstance(t1, type): + continue + if any(isinstance(t2, type) and issubclass(t1, t2) + for t2 in all_params - {t1}): + all_params.remove(t1) + return tuple(t for t in params if t in all_params) - def __repr__(self): - qname = _trim_name(_qualname(self)) - return '%s.%s' % (self.__module__, qname) + +def _check_generic(cls, parameters): + # Check correct count for parameters of a generic cls (internal helper). + if not cls.__parameters__: + raise TypeError("%s is not a generic class" % repr(cls)) + alen = len(parameters) + elen = len(cls.__parameters__) + if alen != elen: + raise TypeError("Too %s parameters for %s; actual %s, expected %s" % + ("many" if alen > elen else "few", repr(cls), alen, elen)) + + +_cleanups = [] -class _TypingBase(metaclass=TypingMeta, _root=True): +def _tp_cache(func): + """Internal wrapper caching __getitem__ of generic types with a fallback to + original function for non-hashable arguments. + """ + + cached = functools.lru_cache()(func) + _cleanups.append(cached.cache_clear) + + @functools.wraps(func) + def inner(*args, **kwds): + try: + return cached(*args, **kwds) + except TypeError: + pass # All real errors (not unhashable args) are raised below. + return func(*args, **kwds) + return inner + +# +# Internal marker base classes: _TypingBase, _FinalTypingBase, _SingletonTypingBase +# + +class _TypingBase: """Internal indicator of special typing constructs.""" __slots__ = ('__weakref__',) - def __init__(self, *args, **kwds): - pass - def __new__(cls, *args, **kwds): """Constructor. @@ -178,7 +259,6 @@ def __new__(cls, *args, **kwds): raise TypeError("Cannot subclass %r" % cls) return super().__new__(cls) - # Things that are not classes also need these. def _eval_type(self, globalns, localns): return self @@ -194,7 +274,20 @@ def __call__(self, *args, **kwds): raise TypeError("Cannot instantiate %r" % type(self)) -class _FinalTypingBase(_TypingBase, _root=True): +class _FinalTypingBase(_TypingBase): + + def __init_subclass__(self, *args, **kwds): + if not kwds.pop('_root', False): + raise TypeError("Cannot subclass special typing classes") + + def __instancecheck__(self, obj): + raise TypeError("%r cannot be used with isinstance()." % self) + + def __subclasscheck__(self, cls): + raise TypeError("%r cannot be used with issubclass()." % self) + + +class _SingletonTypingBase(_FinalTypingBase, _root=True): """Internal mix-in class to prevent instantiation. Prevents instantiation unless _root=True is given in class call. @@ -212,8 +305,12 @@ def __new__(cls, *args, _root=False, **kwds): def __reduce__(self): return _trim_name(type(self).__name__) +# +# Final classes: ForwardRef and TypeVar. These should not be subclassed, +# but can be instantiated. +# -class _ForwardRef(_TypingBase, _root=True): +class _ForwardRef(_FinalTypingBase, _root=True): """Internal wrapper to hold a forward reference.""" __slots__ = ('__forward_arg__', '__forward_code__', @@ -247,210 +344,11 @@ def _eval_type(self, globalns, localns): self.__forward_evaluated__ = True return self.__forward_value__ - def __eq__(self, other): - if not isinstance(other, _ForwardRef): - return NotImplemented - return (self.__forward_arg__ == other.__forward_arg__ and - self.__forward_value__ == other.__forward_value__) - - def __hash__(self): - return hash((self.__forward_arg__, self.__forward_value__)) - - def __instancecheck__(self, obj): - raise TypeError("Forward references cannot be used with isinstance().") - - def __subclasscheck__(self, cls): - raise TypeError("Forward references cannot be used with issubclass().") - def __repr__(self): return '_ForwardRef(%r)' % (self.__forward_arg__,) -class _TypeAlias(_TypingBase, _root=True): - """Internal helper class for defining generic variants of concrete types. - - Note that this is not a type; let's call it a pseudo-type. It cannot - be used in instance and subclass checks in parameterized form, i.e. - ``isinstance(42, Match[str])`` raises ``TypeError`` instead of returning - ``False``. - """ - - __slots__ = ('name', 'type_var', 'impl_type', 'type_checker') - - def __init__(self, name, type_var, impl_type, type_checker): - """Initializer. - - Args: - name: The name, e.g. 'Pattern'. - type_var: The type parameter, e.g. AnyStr, or the - specific type, e.g. str. - impl_type: The implementation type. - type_checker: Function that takes an impl_type instance. - and returns a value that should be a type_var instance. - """ - assert isinstance(name, str), repr(name) - assert isinstance(impl_type, type), repr(impl_type) - assert not isinstance(impl_type, TypingMeta), repr(impl_type) - assert isinstance(type_var, (type, _TypingBase)), repr(type_var) - self.name = name - self.type_var = type_var - self.impl_type = impl_type - self.type_checker = type_checker - - def __repr__(self): - return "%s[%s]" % (self.name, _type_repr(self.type_var)) - - def __getitem__(self, parameter): - if not isinstance(self.type_var, TypeVar): - raise TypeError("%s cannot be further parameterized." % self) - if self.type_var.__constraints__ and isinstance(parameter, type): - if not issubclass(parameter, self.type_var.__constraints__): - raise TypeError("%s is not a valid substitution for %s." % - (parameter, self.type_var)) - if isinstance(parameter, TypeVar) and parameter is not self.type_var: - raise TypeError("%s cannot be re-parameterized." % self) - return self.__class__(self.name, parameter, - self.impl_type, self.type_checker) - - def __eq__(self, other): - if not isinstance(other, _TypeAlias): - return NotImplemented - return self.name == other.name and self.type_var == other.type_var - - def __hash__(self): - return hash((self.name, self.type_var)) - - def __instancecheck__(self, obj): - if not isinstance(self.type_var, TypeVar): - raise TypeError("Parameterized type aliases cannot be used " - "with isinstance().") - return isinstance(obj, self.impl_type) - - def __subclasscheck__(self, cls): - if not isinstance(self.type_var, TypeVar): - raise TypeError("Parameterized type aliases cannot be used " - "with issubclass().") - return issubclass(cls, self.impl_type) - - -def _get_type_vars(types, tvars): - for t in types: - if isinstance(t, TypingMeta) or isinstance(t, _TypingBase): - t._get_type_vars(tvars) - - -def _type_vars(types): - tvars = [] - _get_type_vars(types, tvars) - return tuple(tvars) - - -def _eval_type(t, globalns, localns): - if isinstance(t, TypingMeta) or isinstance(t, _TypingBase): - return t._eval_type(globalns, localns) - return t - - -def _type_check(arg, msg): - """Check that the argument is a type, and return it (internal helper). - - As a special case, accept None and return type(None) instead. - Also, _TypeAlias instances (e.g. Match, Pattern) are acceptable. - - The msg argument is a human-readable error message, e.g. - - "Union[arg, ...]: arg should be a type." - - We append the repr() of the actual value (truncated to 100 chars). - """ - if arg is None: - return type(None) - if isinstance(arg, str): - arg = _ForwardRef(arg) - if ( - isinstance(arg, _TypingBase) and type(arg).__name__ == '_ClassVar' or - not isinstance(arg, (type, _TypingBase)) and not callable(arg) - ): - raise TypeError(msg + " Got %.100r." % (arg,)) - # Bare Union etc. are not valid as type arguments - if ( - type(arg).__name__ in ('_Union', '_Optional') and - not getattr(arg, '__origin__', None) or - isinstance(arg, TypingMeta) and arg._gorg in (Generic, _Protocol) - ): - raise TypeError("Plain %s is not valid as type argument" % arg) - return arg - - -def _type_repr(obj): - """Return the repr() of an object, special-casing types (internal helper). - - If obj is a type, we return a shorter version than the default - type.__repr__, based on the module and qualified name, which is - typically enough to uniquely identify a type. For everything - else, we fall back on repr(obj). - """ - if isinstance(obj, type) and not isinstance(obj, TypingMeta): - if obj.__module__ == 'builtins': - return _qualname(obj) - return '%s.%s' % (obj.__module__, _qualname(obj)) - if obj is ...: - return('...') - if isinstance(obj, types.FunctionType): - return obj.__name__ - return repr(obj) - - -class _Any(_FinalTypingBase, _root=True): - """Special type indicating an unconstrained type. - - - Any is compatible with every type. - - Any assumed to have all methods. - - All values assumed to be instances of Any. - - Note that all the above statements are true from the point of view of - static type checkers. At runtime, Any should not be used with instance - or class checks. - """ - - __slots__ = () - - def __instancecheck__(self, obj): - raise TypeError("Any cannot be used with isinstance().") - - def __subclasscheck__(self, cls): - raise TypeError("Any cannot be used with issubclass().") - - -Any = _Any(_root=True) - - -class _NoReturn(_FinalTypingBase, _root=True): - """Special type indicating functions that never return. - Example:: - - from typing import NoReturn - - def stop() -> NoReturn: - raise Exception('no way') - - This type is invalid in other positions, e.g., ``List[NoReturn]`` - will fail in static type checkers. - """ - - __slots__ = () - - def __instancecheck__(self, obj): - raise TypeError("NoReturn cannot be used with isinstance().") - - def __subclasscheck__(self, cls): - raise TypeError("NoReturn cannot be used with issubclass().") - - -NoReturn = _NoReturn(_root=True) - - -class TypeVar(_TypingBase, _root=True): +class TypeVar(_FinalTypingBase, _root=True): """Type variable. Usage:: @@ -497,8 +395,6 @@ def longest(x: A, y: A) -> A: def __init__(self, name, *constraints, bound=None, covariant=False, contravariant=False): - super().__init__(name, *constraints, bound=bound, - covariant=covariant, contravariant=contravariant) self.__name__ = name if covariant and contravariant: raise ValueError("Bivariant types are not supported.") @@ -528,12 +424,6 @@ def __repr__(self): prefix = '~' return prefix + self.__name__ - def __instancecheck__(self, instance): - raise TypeError("Type variables cannot be used with isinstance().") - - def __subclasscheck__(self, cls): - raise TypeError("Type variables cannot be used with issubclass().") - # Some unconstrained type variables. These are used by the container types. # (These are not for export.) @@ -549,141 +439,83 @@ def __subclasscheck__(self, cls): # (This one *is* for export!) AnyStr = TypeVar('AnyStr', bytes, str) +# +# Singleton classes: Any and NoReturn. +# These should not be neither subclassed, nor instantiated. +# -def _replace_arg(arg, tvars, args): - """An internal helper function: replace arg if it is a type variable - found in tvars with corresponding substitution from args or - with corresponding substitution sub-tree if arg is a generic type. +class _Any(_SingletonTypingBase, _root=True): + """Special type indicating an unconstrained type. + + - Any is compatible with every type. + - Any assumed to have all methods. + - All values assumed to be instances of Any. + + Note that all the above statements are true from the point of view of + static type checkers. At runtime, Any should not be used with instance + or class checks. """ - if tvars is None: - tvars = [] - if hasattr(arg, '_subs_tree') and isinstance(arg, (GenericMeta, _TypingBase)): - return arg._subs_tree(tvars, args) - if isinstance(arg, TypeVar): - for i, tvar in enumerate(tvars): - if arg == tvar: - return args[i] - return arg + __slots__ = () -# Special typing constructs Union, Optional, Generic, Callable and Tuple -# use three special attributes for internal bookkeeping of generic types: -# * __parameters__ is a tuple of unique free type parameters of a generic -# type, for example, Dict[T, T].__parameters__ == (T,); -# * __origin__ keeps a reference to a type that was subscripted, -# e.g., Union[T, int].__origin__ == Union; -# * __args__ is a tuple of all arguments used in subscripting, -# e.g., Dict[T, int].__args__ == (T, int). +Any = _Any(_root=True) + + +class _NoReturn(_SingletonTypingBase, _root=True): + """Special type indicating functions that never return. + Example:: + from typing import NoReturn -def _subs_tree(cls, tvars=None, args=None): - """An internal helper function: calculate substitution tree - for generic cls after replacing its type parameters with - substitutions in tvars -> args (if any). - Repeat the same following __origin__'s. + def stop() -> NoReturn: + raise Exception('no way') - Return a list of arguments with all possible substitutions - performed. Arguments that are generic classes themselves are represented - as tuples (so that no new classes are created by this function). - For example: _subs_tree(List[Tuple[int, T]][str]) == [(Tuple, int, str)] + This type is invalid in other positions, e.g., ``List[NoReturn]`` + will fail in static type checkers. """ - if cls.__origin__ is None: - return cls - # Make of chain of origins (i.e. cls -> cls.__origin__) - current = cls.__origin__ - orig_chain = [] - while current.__origin__ is not None: - orig_chain.append(current) - current = current.__origin__ - # Replace type variables in __args__ if asked ... - tree_args = [] - for arg in cls.__args__: - tree_args.append(_replace_arg(arg, tvars, args)) - # ... then continue replacing down the origin chain. - for ocls in orig_chain: - new_tree_args = [] - for arg in ocls.__args__: - new_tree_args.append(_replace_arg(arg, ocls.__parameters__, tree_args)) - tree_args = new_tree_args - return tree_args + __slots__ = () -def _remove_dups_flatten(parameters): - """An internal helper for Union creation and substitution: flatten Union's - among parameters, then remove duplicates and strict subclasses. - """ +NoReturn = _NoReturn(_root=True) - # Flatten out Union[Union[...], ...]. - params = [] - for p in parameters: - if isinstance(p, _Union) and p.__origin__ is Union: - params.extend(p.__args__) - elif isinstance(p, tuple) and len(p) > 0 and p[0] is Union: - params.extend(p[1:]) - else: - params.append(p) - # Weed out strict duplicates, preserving the first of each occurrence. - all_params = set(params) - if len(all_params) < len(params): - new_params = [] - for t in params: - if t in all_params: - new_params.append(t) - all_params.remove(t) - params = new_params - assert not all_params, all_params - # Weed out subclasses. - # E.g. Union[int, Employee, Manager] == Union[int, Employee]. - # If object is present it will be sole survivor among proper classes. - # Never discard type variables. - # (In particular, Union[str, AnyStr] != AnyStr.) - all_params = set(params) - for t1 in params: - if not isinstance(t1, type): - continue - if any(isinstance(t2, type) and issubclass(t1, t2) - for t2 in all_params - {t1} - if not (isinstance(t2, GenericMeta) and - t2.__origin__ is not None)): - all_params.remove(t1) - return tuple(t for t in params if t in all_params) +# +# Subscriptable singleton classes: ClassVar, Union, and Optional. +# Like above, but can be subscripted. +# -def _check_generic(cls, parameters): - # Check correct count for parameters of a generic cls (internal helper). - if not cls.__parameters__: - raise TypeError("%s is not a generic class" % repr(cls)) - alen = len(parameters) - elen = len(cls.__parameters__) - if alen != elen: - raise TypeError("Too %s parameters for %s; actual %s, expected %s" % - ("many" if alen > elen else "few", repr(cls), alen, elen)) +class _ClassVar(_SingletonTypingBase, _root=True): + """Special type construct to mark class variables. -_cleanups = [] + An annotation wrapped in ClassVar indicates that a given + attribute is intended to be used as a class variable and + should not be set on instances of that class. Usage:: + class Starship: + stats: ClassVar[Dict[str, int]] = {} # class variable + damage: int = 10 # instance variable -def _tp_cache(func): - """Internal wrapper caching __getitem__ of generic types with a fallback to - original function for non-hashable arguments. + ClassVar accepts only types and cannot be further subscribed. + + Note that ClassVar is not a class itself, and should not + be used with isinstance() or issubclass(). """ - cached = functools.lru_cache()(func) - _cleanups.append(cached.cache_clear) + __slots__ = () - @functools.wraps(func) - def inner(*args, **kwds): - try: - return cached(*args, **kwds) - except TypeError: - pass # All real errors (not unhashable args) are raised below. - return func(*args, **kwds) - return inner + @_tp_cache + def __getitem__(self, item): + item = _type_check(item, 'ClassVar accepts only single type.') + return _GenericAlias(self, (item,)) -class _Union(_FinalTypingBase, _root=True): +ClassVar = _ClassVar(_root=True) + + +class _Union(_SingletonTypingBase, _root=True): """Union type; Union[X, Y] means either X or Y. To define a union, use e.g. Union[int, str]. Details: @@ -727,65 +559,7 @@ class Manager(Employee): pass - You can use Optional[X] as a shorthand for Union[X, None]. """ - __slots__ = ('__parameters__', '__args__', '__origin__', '__tree_hash__') - - def __new__(cls, parameters=None, origin=None, *args, _root=False): - self = super().__new__(cls, parameters, origin, *args, _root=_root) - if origin is None: - self.__parameters__ = None - self.__args__ = None - self.__origin__ = None - self.__tree_hash__ = hash(frozenset(('Union',))) - return self - if not isinstance(parameters, tuple): - raise TypeError("Expected parameters=") - if origin is Union: - parameters = _remove_dups_flatten(parameters) - # It's not a union if there's only one type left. - if len(parameters) == 1: - return parameters[0] - self.__parameters__ = _type_vars(parameters) - self.__args__ = parameters - self.__origin__ = origin - # Pre-calculate the __hash__ on instantiation. - # This improves speed for complex substitutions. - subs_tree = self._subs_tree() - if isinstance(subs_tree, tuple): - self.__tree_hash__ = hash(frozenset(subs_tree)) - else: - self.__tree_hash__ = hash(subs_tree) - return self - - def _eval_type(self, globalns, localns): - if self.__args__ is None: - return self - ev_args = tuple(_eval_type(t, globalns, localns) for t in self.__args__) - ev_origin = _eval_type(self.__origin__, globalns, localns) - if ev_args == self.__args__ and ev_origin == self.__origin__: - # Everything is already evaluated. - return self - return self.__class__(ev_args, ev_origin, _root=True) - - def _get_type_vars(self, tvars): - if self.__origin__ and self.__parameters__: - _get_type_vars(self.__parameters__, tvars) - - def __repr__(self): - if self.__origin__ is None: - return super().__repr__() - tree = self._subs_tree() - if not isinstance(tree, tuple): - return repr(tree) - return tree[0]._tree_repr(tree) - - def _tree_repr(self, tree): - arg_list = [] - for arg in tree[1:]: - if not isinstance(arg, tuple): - arg_list.append(_type_repr(arg)) - else: - arg_list.append(arg[0]._tree_repr(arg)) - return super().__repr__() + '[%s]' % ', '.join(arg_list) + __slots__ = () @_tp_cache def __getitem__(self, parameters): @@ -793,46 +567,18 @@ def __getitem__(self, parameters): raise TypeError("Cannot take a Union of no types.") if not isinstance(parameters, tuple): parameters = (parameters,) - if self.__origin__ is None: - msg = "Union[arg, ...]: each arg must be a type." - else: - msg = "Parameters to generic types must be types." + msg = "Union[arg, ...]: each arg must be a type." parameters = tuple(_type_check(p, msg) for p in parameters) - if self is not Union: - _check_generic(self, parameters) - return self.__class__(parameters, origin=self, _root=True) - - def _subs_tree(self, tvars=None, args=None): - if self is Union: - return Union # Nothing to substitute - tree_args = _subs_tree(self, tvars, args) - tree_args = _remove_dups_flatten(tree_args) - if len(tree_args) == 1: - return tree_args[0] # Union of a single type is that type - return (Union,) + tree_args - - def __eq__(self, other): - if isinstance(other, _Union): - return self.__tree_hash__ == other.__tree_hash__ - elif self is not Union: - return self._subs_tree() == other - else: - return self is other - - def __hash__(self): - return self.__tree_hash__ - - def __instancecheck__(self, obj): - raise TypeError("Unions cannot be used with isinstance().") - - def __subclasscheck__(self, cls): - raise TypeError("Unions cannot be used with issubclass().") + parameters = _remove_dups_flatten(parameters) + if len(parameters) == 1: + return parameters[0] + return _GenericAlias(self, parameters) Union = _Union(_root=True) -class _Optional(_FinalTypingBase, _root=True): +class _Optional(_SingletonTypingBase, _root=True): """Optional type. Optional[X] is equivalent to Union[X, None]. @@ -849,199 +595,25 @@ def __getitem__(self, arg): Optional = _Optional(_root=True) -def _next_in_mro(cls): - """Helper for Generic.__new__. - - Returns the class after the last occurrence of Generic or - Generic[...] in cls.__mro__. - """ - next_in_mro = object - # Look for the last occurrence of Generic or Generic[...]. - for i, c in enumerate(cls.__mro__[:-1]): - if isinstance(c, GenericMeta) and c._gorg is Generic: - next_in_mro = cls.__mro__[i + 1] - return next_in_mro - - -def _make_subclasshook(cls): - """Construct a __subclasshook__ callable that incorporates - the associated __extra__ class in subclass checks performed - against cls. - """ - if isinstance(cls.__extra__, abc.ABCMeta): - # The logic mirrors that of ABCMeta.__subclasscheck__. - # Registered classes need not be checked here because - # cls and its extra share the same _abc_registry. - def __extrahook__(subclass): - res = cls.__extra__.__subclasshook__(subclass) - if res is not NotImplemented: - return res - if cls.__extra__ in subclass.__mro__: - return True - for scls in cls.__extra__.__subclasses__(): - if isinstance(scls, GenericMeta): - continue - if issubclass(subclass, scls): - return True - return NotImplemented - else: - # For non-ABC extras we'll just call issubclass(). - def __extrahook__(subclass): - if cls.__extra__ and issubclass(subclass, cls.__extra__): - return True - return NotImplemented - return __extrahook__ - +# Special typing constructs Union, Optional, Generic, Callable and Tuple +# use three special attributes for internal bookkeeping of generic types: +# * __parameters__ is a tuple of unique free type parameters of a generic +# type, for example, Dict[T, T].__parameters__ == (T,); +# * __origin__ keeps a reference to a type that was subscripted, +# e.g., Union[T, int].__origin__ == Union; +# * __args__ is a tuple of all arguments used in subscripting, +# e.g., Dict[T, int].__args__ == (T, int). -def _no_slots_copy(dct): - """Internal helper: copy class __dict__ and clean slots class variables. - (They will be re-created if necessary by normal class machinery.) - """ - dict_copy = dict(dct) - if '__slots__' in dict_copy: - for slot in dict_copy['__slots__']: - dict_copy.pop(slot, None) - return dict_copy - - -class GenericMeta(TypingMeta, abc.ABCMeta): - """Metaclass for generic types. - - This is a metaclass for typing.Generic and generic ABCs defined in - typing module. User defined subclasses of GenericMeta can override - __new__ and invoke super().__new__. Note that GenericMeta.__new__ - has strict rules on what is allowed in its bases argument: - * plain Generic is disallowed in bases; - * Generic[...] should appear in bases at most once; - * if Generic[...] is present, then it should list all type variables - that appear in other bases. - In addition, type of all generic bases is erased, e.g., C[int] is - stripped to plain C. - """ - def __new__(cls, name, bases, namespace, - tvars=None, args=None, origin=None, extra=None, orig_bases=None): +class _GenericAlias(_FinalTypingBase, _root=True): + def __init__(self, origin, params): """Create a new generic class. GenericMeta.__new__ accepts keyword arguments that are used for internal bookkeeping, therefore an override should pass unused keyword arguments to super(). """ - if tvars is not None: - # Called from __getitem__() below. - assert origin is not None - assert all(isinstance(t, TypeVar) for t in tvars), tvars - else: - # Called from class statement. - assert tvars is None, tvars - assert args is None, args - assert origin is None, origin - - # Get the full set of tvars from the bases. - tvars = _type_vars(bases) - # Look for Generic[T1, ..., Tn]. - # If found, tvars must be a subset of it. - # If not found, tvars is it. - # Also check for and reject plain Generic, - # and reject multiple Generic[...]. - gvars = None - for base in bases: - if base is Generic: - raise TypeError("Cannot inherit from plain Generic") - if (isinstance(base, GenericMeta) and - base.__origin__ is Generic): - if gvars is not None: - raise TypeError( - "Cannot inherit from Generic[...] multiple types.") - gvars = base.__parameters__ - if gvars is None: - gvars = tvars - else: - tvarset = set(tvars) - gvarset = set(gvars) - if not tvarset <= gvarset: - raise TypeError( - "Some type variables (%s) " - "are not listed in Generic[%s]" % - (", ".join(str(t) for t in tvars if t not in gvarset), - ", ".join(str(g) for g in gvars))) - tvars = gvars - - initial_bases = bases - if extra is not None and type(extra) is abc.ABCMeta and extra not in bases: - bases = (extra,) + bases - bases = tuple(b._gorg if isinstance(b, GenericMeta) else b for b in bases) - - # remove bare Generic from bases if there are other generic bases - if any(isinstance(b, GenericMeta) and b is not Generic for b in bases): - bases = tuple(b for b in bases if b is not Generic) - namespace.update({'__origin__': origin, '__extra__': extra}) - self = super().__new__(cls, name, bases, namespace, _root=True) - super(GenericMeta, self).__setattr__('_gorg', - self if not origin else origin._gorg) - self.__parameters__ = tvars - # Be prepared that GenericMeta will be subclassed by TupleMeta - # and CallableMeta, those two allow ..., (), or [] in __args___. - self.__args__ = tuple(... if a is _TypingEllipsis else - () if a is _TypingEmpty else - a for a in args) if args else None - # Speed hack (https://github.com/python/typing/issues/196). - self.__next_in_mro__ = _next_in_mro(self) - # Preserve base classes on subclassing (__bases__ are type erased now). - if orig_bases is None: - self.__orig_bases__ = initial_bases - - # This allows unparameterized generic collections to be used - # with issubclass() and isinstance() in the same way as their - # collections.abc counterparts (e.g., isinstance([], Iterable)). - if ( - '__subclasshook__' not in namespace and extra or - # allow overriding - getattr(self.__subclasshook__, '__name__', '') == '__extrahook__' - ): - self.__subclasshook__ = _make_subclasshook(self) - if isinstance(extra, abc.ABCMeta): - self._abc_registry = extra._abc_registry - self._abc_cache = extra._abc_cache - elif origin is not None: - self._abc_registry = origin._abc_registry - self._abc_cache = origin._abc_cache - - if origin and hasattr(origin, '__qualname__'): # Fix for Python 3.2. - self.__qualname__ = origin.__qualname__ - self.__tree_hash__ = (hash(self._subs_tree()) if origin else - super(GenericMeta, self).__hash__()) - return self - - # _abc_negative_cache and _abc_negative_cache_version - # realised as descriptors, since GenClass[t1, t2, ...] always - # share subclass info with GenClass. - # This is an important memory optimization. - @property - def _abc_negative_cache(self): - if isinstance(self.__extra__, abc.ABCMeta): - return self.__extra__._abc_negative_cache - return self._gorg._abc_generic_negative_cache - - @_abc_negative_cache.setter - def _abc_negative_cache(self, value): - if self.__origin__ is None: - if isinstance(self.__extra__, abc.ABCMeta): - self.__extra__._abc_negative_cache = value - else: - self._abc_generic_negative_cache = value - - @property - def _abc_negative_cache_version(self): - if isinstance(self.__extra__, abc.ABCMeta): - return self.__extra__._abc_negative_cache_version - return self._gorg._abc_generic_negative_cache_version - - @_abc_negative_cache_version.setter - def _abc_negative_cache_version(self, value): - if self.__origin__ is None: - if isinstance(self.__extra__, abc.ABCMeta): - self.__extra__._abc_negative_cache_version = value - else: - self._abc_generic_negative_cache_version = value + self.__origin__ = origin + self.__args__ = params + self.__parameters__ = _type_vars(params) def _get_type_vars(self, tvars): if self.__origin__ and self.__parameters__: @@ -1054,150 +626,27 @@ def _eval_type(self, globalns, localns): in self.__args__) if self.__args__ else None if ev_origin == self.__origin__ and ev_args == self.__args__: return self - return self.__class__(self.__name__, - self.__bases__, - _no_slots_copy(self.__dict__), - tvars=_type_vars(ev_args) if ev_args else None, - args=ev_args, - origin=ev_origin, - extra=self.__extra__, - orig_bases=self.__orig_bases__) - - def __repr__(self): - if self.__origin__ is None: - return super().__repr__() - return self._tree_repr(self._subs_tree()) - - def _tree_repr(self, tree): - arg_list = [] - for arg in tree[1:]: - if arg == (): - arg_list.append('()') - elif not isinstance(arg, tuple): - arg_list.append(_type_repr(arg)) - else: - arg_list.append(arg[0]._tree_repr(arg)) - return super().__repr__() + '[%s]' % ', '.join(arg_list) - - def _subs_tree(self, tvars=None, args=None): - if self.__origin__ is None: - return self - tree_args = _subs_tree(self, tvars, args) - return (self._gorg,) + tuple(tree_args) - - def __eq__(self, other): - if not isinstance(other, GenericMeta): - return NotImplemented - if self.__origin__ is None or other.__origin__ is None: - return self is other - return self.__tree_hash__ == other.__tree_hash__ - - def __hash__(self): - return self.__tree_hash__ + return _GenericAlias() @_tp_cache - def __getitem__(self, params): - if not isinstance(params, tuple): - params = (params,) - if not params and self._gorg is not Tuple: - raise TypeError( - "Parameter list to %s[...] cannot be empty" % _qualname(self)) - msg = "Parameters to generic types must be types." - params = tuple(_type_check(p, msg) for p in params) - if self is Generic: - # Generic can only be subscripted with unique type variables. - if not all(isinstance(p, TypeVar) for p in params): - raise TypeError( - "Parameters to Generic[...] must all be type variables") - if len(set(params)) != len(params): - raise TypeError( - "Parameters to Generic[...] must all be unique") - tvars = params - args = params - elif self in (Tuple, Callable): - tvars = _type_vars(params) - args = params - elif self is _Protocol: - # _Protocol is internal, don't check anything. - tvars = params - args = params - elif self.__origin__ in (Generic, _Protocol): + def __getitem__(self, parameters): + if self.__origin__ in (Generic, _Protocol): # Can't subscript Generic[...] or _Protocol[...]. raise TypeError("Cannot subscript already-subscripted %s" % repr(self)) - else: - # Subscripting a regular Generic subclass. - _check_generic(self, params) - tvars = _type_vars(params) - args = params - - prepend = (self,) if self.__origin__ is None else () - return self.__class__(self.__name__, - prepend + self.__bases__, - _no_slots_copy(self.__dict__), - tvars=tvars, - args=args, - origin=self, - extra=self.__extra__, - orig_bases=self.__orig_bases__) - - def __subclasscheck__(self, cls): - if self.__origin__ is not None: - if sys._getframe(1).f_globals['__name__'] not in ['abc', 'functools']: - raise TypeError("Parameterized generics cannot be used with class " - "or instance checks") - return False - if self is Generic: - raise TypeError("Class %r cannot be used with class " - "or instance checks" % self) - return super().__subclasscheck__(cls) - - def __instancecheck__(self, instance): - # Since we extend ABC.__subclasscheck__ and - # ABC.__instancecheck__ inlines the cache checking done by the - # latter, we must extend __instancecheck__ too. For simplicity - # we just skip the cache check -- instance checks for generic - # classes are supposed to be rare anyways. - return issubclass(instance.__class__, self) - - def __copy__(self): - return self.__class__(self.__name__, self.__bases__, - _no_slots_copy(self.__dict__), - self.__parameters__, self.__args__, self.__origin__, - self.__extra__, self.__orig_bases__) - - def __setattr__(self, attr, value): - # We consider all the subscripted generics as proxies for original class - if ( - attr.startswith('__') and attr.endswith('__') or - attr.startswith('_abc_') - ): - super(GenericMeta, self).__setattr__(attr, value) - else: - super(GenericMeta, self._gorg).__setattr__(attr, value) - -# Prevent checks for Generic to crash when defining Generic. -Generic = None + def __repr__(self): + if self.__origin__ is None: + return super().__repr__() + return self._tree_repr(self._subs_tree()) + def __mro_entry__(self, bases): + return self.__origin__ -def _generic_new(base_cls, cls, *args, **kwds): - # Assure type is erased on instantiation, - # but attempt to store it in __orig_class__ - if cls.__origin__ is None: - return base_cls.__new__(cls) - else: - origin = cls._gorg - obj = base_cls.__new__(origin) - try: - obj.__orig_class__ = cls - except AttributeError: - pass - obj.__init__(*args, **kwds) - return obj + # TODO: __getattr__ and __setattr__. -class Generic(metaclass=GenericMeta): +class Generic(_TypingBase): """Abstract base class for generic types. A generic type is typically declared by inheriting from @@ -1221,10 +670,49 @@ def lookup_name(mapping: Mapping[KT, VT], key: KT, default: VT) -> VT: __slots__ = () def __new__(cls, *args, **kwds): - if cls._gorg is Generic: + if cls is Generic: raise TypeError("Type Generic cannot be instantiated; " "it can be used only as a base class") - return _generic_new(cls.__next_in_mro__, cls, *args, **kwds) + return super().__new__(cls, *args, **kwds) + + @_tp_cache + def __class_getitem__(cls, params): + if not isinstance(params, tuple): + params = (params,) + if not params and cls is not Tuple: + raise TypeError( + "Parameter list to %s[...] cannot be empty" % cls.__qualname__) + msg = "Parameters to generic types must be types." + params = tuple(_type_check(p, msg) for p in params) + if cls is Generic: + # Generic can only be subscripted with unique type variables. + if not all(isinstance(p, TypeVar) for p in params): + raise TypeError( + "Parameters to Generic[...] must all be type variables") + if len(set(params)) != len(params): + raise TypeError( + "Parameters to Generic[...] must all be unique") + tvars = params + args = params + elif cls in (Tuple, Callable): + tvars = _type_vars(params) + args = params + elif cls is _Protocol: + # _Protocol is internal, don't check anything. + tvars = params + args = params + else: + # Subscripting a regular Generic subclass. + _check_generic(cls, params) + tvars = _type_vars(params) + args = params + return _GenericAlias(cls, params) + + def __init_subclass__(cls, *args, **kwargs): + pars = [] + if hasattr(cls, '__orig_bases__'): + pars = _type_vars(cls.__orig_bases__) + cls.__parameters__ = tuple(pars) class _TypingEmpty: @@ -1238,11 +726,26 @@ class _TypingEllipsis: """Internal placeholder for ... (ellipsis).""" -class TupleMeta(GenericMeta): - """Metaclass for Tuple (internal).""" +class Tuple(tuple, Generic): + """Tuple type; Tuple[X, Y] is the cross-product type of X and Y. + + Example: Tuple[T1, T2] is a tuple of two elements corresponding + to type variables T1 and T2. Tuple[int, float, str] is a tuple + of an int, a float and a string. + + To specify a variable-length tuple of homogeneous type, use Tuple[T, ...]. + """ + + __slots__ = () + + def __new__(cls, *args, **kwds): + if cls is Tuple: + raise TypeError("Type Tuple cannot be instantiated; " + "use tuple() instead") + return super().__new__(cls, *args, **kwds) @_tp_cache - def __getitem__(self, parameters): + def __class_getitem__(self, parameters): if self.__origin__ is not None or self._gorg is not Tuple: # Normal generic rules apply if this is not the first subscription # or a subscription of a subclass. @@ -1259,63 +762,27 @@ def __getitem__(self, parameters): parameters = tuple(_type_check(p, msg) for p in parameters) return super().__getitem__(parameters) - def __instancecheck__(self, obj): - if self.__args__ is None: - return isinstance(obj, tuple) - raise TypeError("Parameterized Tuple cannot be used " - "with isinstance().") - - def __subclasscheck__(self, cls): - if self.__args__ is None: - return issubclass(cls, tuple) - raise TypeError("Parameterized Tuple cannot be used " - "with issubclass().") - -class Tuple(tuple, extra=tuple, metaclass=TupleMeta): - """Tuple type; Tuple[X, Y] is the cross-product type of X and Y. +class Callable(collections.abc.Callable, Generic): + """Callable type; Callable[[int], str] is a function of (int) -> str. - Example: Tuple[T1, T2] is a tuple of two elements corresponding - to type variables T1 and T2. Tuple[int, float, str] is a tuple - of an int, a float and a string. + The subscription syntax must always be used with exactly two + values: the argument list and the return type. The argument list + must be a list of types or ellipsis; the return type must be a single type. - To specify a variable-length tuple of homogeneous type, use Tuple[T, ...]. + There is no syntax to indicate optional or keyword arguments, + such function types are rarely used as callback types. """ __slots__ = () def __new__(cls, *args, **kwds): - if cls._gorg is Tuple: - raise TypeError("Type Tuple cannot be instantiated; " - "use tuple() instead") - return _generic_new(tuple, cls, *args, **kwds) - - -class CallableMeta(GenericMeta): - """Metaclass for Callable (internal).""" - - def __repr__(self): - if self.__origin__ is None: - return super().__repr__() - return self._tree_repr(self._subs_tree()) - - def _tree_repr(self, tree): - if self._gorg is not Callable: - return super()._tree_repr(tree) - # For actual Callable (not its subclass) we override - # super()._tree_repr() for nice formatting. - arg_list = [] - for arg in tree[1:]: - if not isinstance(arg, tuple): - arg_list.append(_type_repr(arg)) - else: - arg_list.append(arg[0]._tree_repr(arg)) - if arg_list[0] == '...': - return repr(tree[0]) + '[..., %s]' % arg_list[1] - return (repr(tree[0]) + - '[[%s], %s]' % (', '.join(arg_list[:-1]), arg_list[-1])) + if cls is Callable: + raise TypeError("Type Callable cannot be instantiated; " + "use a non-abstract subclass instead") + return super().__new__(cls, *args, **kwds) - def __getitem__(self, parameters): + def __class_getitem__(self, parameters): """A thin wrapper around __getitem_inner__ to provide the latter with hashable arguments to improve speed. """ @@ -1348,83 +815,6 @@ def __getitem_inner__(self, parameters): return super().__getitem__(parameters) -class Callable(extra=collections_abc.Callable, metaclass=CallableMeta): - """Callable type; Callable[[int], str] is a function of (int) -> str. - - The subscription syntax must always be used with exactly two - values: the argument list and the return type. The argument list - must be a list of types or ellipsis; the return type must be a single type. - - There is no syntax to indicate optional or keyword arguments, - such function types are rarely used as callback types. - """ - - __slots__ = () - - def __new__(cls, *args, **kwds): - if cls._gorg is Callable: - raise TypeError("Type Callable cannot be instantiated; " - "use a non-abstract subclass instead") - return _generic_new(cls.__next_in_mro__, cls, *args, **kwds) - - -class _ClassVar(_FinalTypingBase, _root=True): - """Special type construct to mark class variables. - - An annotation wrapped in ClassVar indicates that a given - attribute is intended to be used as a class variable and - should not be set on instances of that class. Usage:: - - class Starship: - stats: ClassVar[Dict[str, int]] = {} # class variable - damage: int = 10 # instance variable - - ClassVar accepts only types and cannot be further subscribed. - - Note that ClassVar is not a class itself, and should not - be used with isinstance() or issubclass(). - """ - - __slots__ = ('__type__',) - - def __init__(self, tp=None, **kwds): - self.__type__ = tp - - def __getitem__(self, item): - cls = type(self) - if self.__type__ is None: - return cls(_type_check(item, - '{} accepts only single type.'.format(cls.__name__[1:])), - _root=True) - raise TypeError('{} cannot be further subscripted' - .format(cls.__name__[1:])) - - def _eval_type(self, globalns, localns): - new_tp = _eval_type(self.__type__, globalns, localns) - if new_tp == self.__type__: - return self - return type(self)(new_tp, _root=True) - - def __repr__(self): - r = super().__repr__() - if self.__type__ is not None: - r += '[{}]'.format(_type_repr(self.__type__)) - return r - - def __hash__(self): - return hash((type(self).__name__, self.__type__)) - - def __eq__(self, other): - if not isinstance(other, _ClassVar): - return NotImplemented - if self.__type__ is not None: - return self.__type__ == other.__type__ - return self is other - - -ClassVar = _ClassVar(_root=True) - - def cast(typ, val): """Cast a value to a type. @@ -1623,7 +1013,7 @@ def utf8(value): return _overload_dummy -class _ProtocolMeta(GenericMeta): +class _ProtocolMeta(type): """Internal metaclass for _Protocol. This exists so _Protocol classes can be generic without deriving @@ -1703,46 +1093,36 @@ class _Protocol(metaclass=_ProtocolMeta): _is_protocol = True + def __class_getitem__(cls, params): + return cls + # Various ABCs mimicking those in collections.abc. # A few are simply re-exported for completeness. -Hashable = collections_abc.Hashable # Not generic. - - -if hasattr(collections_abc, 'Awaitable'): - class Awaitable(Generic[T_co], extra=collections_abc.Awaitable): - __slots__ = () - - __all__.append('Awaitable') +Hashable = collections.abc.Hashable # Not generic. -if hasattr(collections_abc, 'Coroutine'): - class Coroutine(Awaitable[V_co], Generic[T_co, T_contra, V_co], - extra=collections_abc.Coroutine): - __slots__ = () - - __all__.append('Coroutine') +class Awaitable(collections.abc.Awaitable, Generic[T_co]): + __slots__ = () -if hasattr(collections_abc, 'AsyncIterable'): +class Coroutine(collections.abc.Coroutine, Generic[T_co, T_contra, V_co]): + __slots__ = () - class AsyncIterable(Generic[T_co], extra=collections_abc.AsyncIterable): - __slots__ = () +class AsyncIterable(collections.abc.AsyncIterable, Generic[T_co]): + __slots__ = () - class AsyncIterator(AsyncIterable[T_co], - extra=collections_abc.AsyncIterator): - __slots__ = () - __all__.append('AsyncIterable') - __all__.append('AsyncIterator') +class AsyncIterator(collections.abc.AsyncIterator, Generic[T_co]): + __slots__ = () -class Iterable(Generic[T_co], extra=collections_abc.Iterable): +class Iterable(collections.abc.Iterable, Generic[T_co]): __slots__ = () -class Iterator(Iterable[T_co], extra=collections_abc.Iterator): +class Iterator(collections.abc.Iterator, Generic[T_co]): __slots__ = () @@ -1794,282 +1174,144 @@ def __round__(self, ndigits: int = 0) -> T_co: pass -if hasattr(collections_abc, 'Reversible'): - class Reversible(Iterable[T_co], extra=collections_abc.Reversible): - __slots__ = () -else: - class Reversible(_Protocol[T_co]): - __slots__ = () - - @abstractmethod - def __reversed__(self) -> 'Iterator[T_co]': - pass +class Reversible(collections.abc.Reversible, Generic[T_co]): + __slots__ = () -Sized = collections_abc.Sized # Not generic. +Sized = collections.abc.Sized # Not generic. -class Container(Generic[T_co], extra=collections_abc.Container): +class Container(collections.abc.Container, Generic[T_co]): __slots__ = () -if hasattr(collections_abc, 'Collection'): - class Collection(Sized, Iterable[T_co], Container[T_co], - extra=collections_abc.Collection): - __slots__ = () - - __all__.append('Collection') +class Collection(collections.abc.Collection, Generic[T_co]): + __slots__ = () # Callable was defined earlier. -if hasattr(collections_abc, 'Collection'): - class AbstractSet(Collection[T_co], - extra=collections_abc.Set): - __slots__ = () -else: - class AbstractSet(Sized, Iterable[T_co], Container[T_co], - extra=collections_abc.Set): - __slots__ = () +class AbstractSet(collections.abc.Set, Generic[T_co]): + __slots__ = () -class MutableSet(AbstractSet[T], extra=collections_abc.MutableSet): +class MutableSet(collections.abc.MutableSet, Generic[T]): __slots__ = () # NOTE: It is only covariant in the value type. -if hasattr(collections_abc, 'Collection'): - class Mapping(Collection[KT], Generic[KT, VT_co], - extra=collections_abc.Mapping): - __slots__ = () -else: - class Mapping(Sized, Iterable[KT], Container[KT], Generic[KT, VT_co], - extra=collections_abc.Mapping): - __slots__ = () - - -class MutableMapping(Mapping[KT, VT], extra=collections_abc.MutableMapping): +class Mapping(collections.abc.Mapping, Generic[KT, VT_co]): __slots__ = () -if hasattr(collections_abc, 'Reversible'): - if hasattr(collections_abc, 'Collection'): - class Sequence(Reversible[T_co], Collection[T_co], - extra=collections_abc.Sequence): - __slots__ = () - else: - class Sequence(Sized, Reversible[T_co], Container[T_co], - extra=collections_abc.Sequence): - __slots__ = () -else: - class Sequence(Sized, Iterable[T_co], Container[T_co], - extra=collections_abc.Sequence): - __slots__ = () +class MutableMapping(collections.abc.MutableMapping, Generic[KT, VT]): + __slots__ = () -class MutableSequence(Sequence[T], extra=collections_abc.MutableSequence): +class Sequence(collections.abc.Sequence, Generic[T_co]): __slots__ = () -class ByteString(Sequence[int], extra=collections_abc.ByteString): +class MutableSequence(collections.abc.MutableSequence, Generic[T]): __slots__ = () -class List(list, MutableSequence[T], extra=list): +# Not generic +ByteString = collections.abc.ByteString + +class List(list, Generic[T]): __slots__ = () def __new__(cls, *args, **kwds): - if cls._gorg is List: + if cls is List: raise TypeError("Type List cannot be instantiated; " "use list() instead") - return _generic_new(list, cls, *args, **kwds) + return super().__new__(cls, *args, **kwds) -class Deque(collections.deque, MutableSequence[T], extra=collections.deque): - +class Deque(collections.deque, Generic[T]): __slots__ = () - def __new__(cls, *args, **kwds): - if cls._gorg is Deque: - return collections.deque(*args, **kwds) - return _generic_new(collections.deque, cls, *args, **kwds) - - -class Set(set, MutableSet[T], extra=set): +class Set(set, Generic[T]): __slots__ = () def __new__(cls, *args, **kwds): - if cls._gorg is Set: + if cls is Set: raise TypeError("Type Set cannot be instantiated; " "use set() instead") - return _generic_new(set, cls, *args, **kwds) + return super().__new__(cls, *args, **kwds) -class FrozenSet(frozenset, AbstractSet[T_co], extra=frozenset): +class FrozenSet(frozenset, Generic[T_co]): __slots__ = () def __new__(cls, *args, **kwds): - if cls._gorg is FrozenSet: + if cls is FrozenSet: raise TypeError("Type FrozenSet cannot be instantiated; " "use frozenset() instead") - return _generic_new(frozenset, cls, *args, **kwds) + return super().__new__(cls, *args, **kwds) -class MappingView(Sized, Iterable[T_co], extra=collections_abc.MappingView): +class MappingView(collections.abc.MappingView, Generic[T_co]): __slots__ = () -class KeysView(MappingView[KT], AbstractSet[KT], - extra=collections_abc.KeysView): +class KeysView(collections.abc.KeysView, Generic[KT]): __slots__ = () -class ItemsView(MappingView[Tuple[KT, VT_co]], - AbstractSet[Tuple[KT, VT_co]], - Generic[KT, VT_co], - extra=collections_abc.ItemsView): +class ItemsView(collections.abc.ItemsView, Generic[KT, VT_co]): __slots__ = () -class ValuesView(MappingView[VT_co], extra=collections_abc.ValuesView): +class ValuesView(collections.abc.ValuesView, Generic[VT_co]): __slots__ = () -if hasattr(contextlib, 'AbstractContextManager'): - class ContextManager(Generic[T_co], extra=contextlib.AbstractContextManager): - __slots__ = () -else: - class ContextManager(Generic[T_co]): - __slots__ = () - - def __enter__(self): - return self - - @abc.abstractmethod - def __exit__(self, exc_type, exc_value, traceback): - return None - - @classmethod - def __subclasshook__(cls, C): - if cls is ContextManager: - # In Python 3.6+, it is possible to set a method to None to - # explicitly indicate that the class does not implement an ABC - # (https://bugs.python.org/issue25958), but we do not support - # that pattern here because this fallback class is only used - # in Python 3.5 and earlier. - if (any("__enter__" in B.__dict__ for B in C.__mro__) and - any("__exit__" in B.__dict__ for B in C.__mro__)): - return True - return NotImplemented - - -if hasattr(contextlib, 'AbstractAsyncContextManager'): - class AsyncContextManager(Generic[T_co], - extra=contextlib.AbstractAsyncContextManager): - __slots__ = () - - __all__.append('AsyncContextManager') -elif sys.version_info[:2] >= (3, 5): - exec(""" -class AsyncContextManager(Generic[T_co]): +class ContextManager(contextlib.AbstractContextManager, Generic[T_co]): __slots__ = () - async def __aenter__(self): - return self - - @abc.abstractmethod - async def __aexit__(self, exc_type, exc_value, traceback): - return None - - @classmethod - def __subclasshook__(cls, C): - if cls is AsyncContextManager: - if sys.version_info[:2] >= (3, 6): - return _collections_abc._check_methods(C, "__aenter__", "__aexit__") - if (any("__aenter__" in B.__dict__ for B in C.__mro__) and - any("__aexit__" in B.__dict__ for B in C.__mro__)): - return True - return NotImplemented -__all__.append('AsyncContextManager') -""") +#class AsyncContextManager(contextlib.AbstractAsyncContextManager, Generic[T_co]): +# __slots__ = () -class Dict(dict, MutableMapping[KT, VT], extra=dict): - +class Dict(dict, Generic[KT, VT]): __slots__ = () def __new__(cls, *args, **kwds): - if cls._gorg is Dict: + if cls is Dict: raise TypeError("Type Dict cannot be instantiated; " "use dict() instead") - return _generic_new(dict, cls, *args, **kwds) - + return super().__new__(cls, *args, **kwds) -class DefaultDict(collections.defaultdict, MutableMapping[KT, VT], - extra=collections.defaultdict): +class DefaultDict(collections.defaultdict, Generic[KT, VT]): __slots__ = () - def __new__(cls, *args, **kwds): - if cls._gorg is DefaultDict: - return collections.defaultdict(*args, **kwds) - return _generic_new(collections.defaultdict, cls, *args, **kwds) - - -class Counter(collections.Counter, Dict[T, int], extra=collections.Counter): +class Counter(collections.Counter, Generic[T]): __slots__ = () - def __new__(cls, *args, **kwds): - if cls._gorg is Counter: - return collections.Counter(*args, **kwds) - return _generic_new(collections.Counter, cls, *args, **kwds) - - -if hasattr(collections, 'ChainMap'): - # ChainMap only exists in 3.3+ - __all__.append('ChainMap') - class ChainMap(collections.ChainMap, MutableMapping[KT, VT], - extra=collections.ChainMap): - - __slots__ = () - - def __new__(cls, *args, **kwds): - if cls._gorg is ChainMap: - return collections.ChainMap(*args, **kwds) - return _generic_new(collections.ChainMap, cls, *args, **kwds) - - -# Determine what base class to use for Generator. -if hasattr(collections_abc, 'Generator'): - # Sufficiently recent versions of 3.5 have a Generator ABC. - _G_base = collections_abc.Generator -else: - # Fall back on the exact type. - _G_base = types.GeneratorType +class ChainMap(collections.ChainMap, Generic[KT, VT]): + __slots__ = () -class Generator(Iterator[T_co], Generic[T_co, T_contra, V_co], - extra=_G_base): +class Generator(collections.abc.Generator, Generic[T_co, T_contra, V_co]): __slots__ = () def __new__(cls, *args, **kwds): - if cls._gorg is Generator: + if cls is Generator: raise TypeError("Type Generator cannot be instantiated; " "create a subclass instead") - return _generic_new(_G_base, cls, *args, **kwds) - + return super().__new__(cls, *args, **kwds) -if hasattr(collections_abc, 'AsyncGenerator'): - class AsyncGenerator(AsyncIterator[T_co], Generic[T_co, T_contra], - extra=collections_abc.AsyncGenerator): - __slots__ = () - __all__.append('AsyncGenerator') +class AsyncGenerator(collections.abc.AsyncGenerator, Generic[T_co, T_contra]): + __slots__ = () # Internal type variable used for Type[]. @@ -2077,7 +1319,7 @@ class AsyncGenerator(AsyncIterator[T_co], Generic[T_co, T_contra], # This is not a real generic class. Don't use outside annotations. -class Type(Generic[CT_co], extra=type): +class Type(type, Generic[CT_co]): """A special construct usable to annotate class objects. For example, suppose we have the following classes:: @@ -2388,11 +1630,8 @@ class io: io.__name__ = __name__ + '.io' sys.modules[io.__name__] = io - -Pattern = _TypeAlias('Pattern', AnyStr, type(stdlib_re.compile('')), - lambda p: p.pattern) -Match = _TypeAlias('Match', AnyStr, type(stdlib_re.match('', '')), - lambda m: m.re.pattern) +Pattern = _GenericAlias(type(stdlib_re.compile('')), (AnyStr,)) +Match = _GenericAlias(type(stdlib_re.match('', '')), (AnyStr,)) class re: From 450d66f81841f46dc974ffa38664aaa650be4d8d Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 12 Nov 2017 17:01:48 +0100 Subject: [PATCH 28/82] More work on new typing --- Lib/test/test_typing.py | 102 +++++++++++++------------------- Lib/typing.py | 126 +++++++++++++++++++++++++++++----------- 2 files changed, 131 insertions(+), 97 deletions(-) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index a3b6eb933935e2..2a3d311a0eec43 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -12,7 +12,7 @@ from typing import Union, Optional from typing import Tuple, List, MutableMapping from typing import Callable -from typing import Generic, ClassVar, GenericMeta +from typing import Generic, ClassVar from typing import cast from typing import get_type_hints from typing import no_type_check, no_type_check_decorator @@ -24,17 +24,8 @@ import abc import typing import weakref -try: - import collections.abc as collections_abc -except ImportError: - import collections as collections_abc # Fallback for PY3.2. - -try: - import mod_generics_cache -except ImportError: - # try to use the builtin one, Python 3.5+ - from test import mod_generics_cache +from test import mod_generics_cache class BaseTestCase(TestCase): @@ -740,7 +731,7 @@ class D(C[T]): def test_abc_registry_kept(self): T = TypeVar('T') - class C(Generic[T]): ... + class C(collections.abc.Mapping, Generic[T]): ... C.register(int) self.assertIsInstance(1, C) C[int] @@ -765,17 +756,17 @@ def __len__(self): return 0 # this should just work MM().update() - self.assertIsInstance(MM(), collections_abc.MutableMapping) + self.assertIsInstance(MM(), collections.abc.MutableMapping) self.assertIsInstance(MM(), MutableMapping) self.assertNotIsInstance(MM(), List) self.assertNotIsInstance({}, MM) def test_multiple_bases(self): - class MM1(MutableMapping[str, str], collections_abc.MutableMapping): + class MM1(MutableMapping[str, str], collections.abc.MutableMapping): pass with self.assertRaises(TypeError): # consistent MRO not possible - class MM2(collections_abc.MutableMapping, MutableMapping[str, str]): + class MM2(collections.abc.MutableMapping, MutableMapping[str, str]): pass def test_orig_bases(self): @@ -838,9 +829,10 @@ class D(C, List[T][U][V]): ... def test_subscript_meta(self): T = TypeVar('T') - self.assertEqual(Type[GenericMeta], Type[GenericMeta]) - self.assertEqual(Union[T, int][GenericMeta], Union[GenericMeta, int]) - self.assertEqual(Callable[..., GenericMeta].__args__, (Ellipsis, GenericMeta)) + class Meta(type): ... + self.assertEqual(Type[Meta], Type[Meta]) + self.assertEqual(Union[T, int][Meta], Union[Meta, int]) + self.assertEqual(Callable[..., Meta].__args__, (Ellipsis, Meta)) def test_generic_hashes(self): class A(Generic[T]): @@ -956,9 +948,9 @@ def __call__(self): self.assertEqual(repr(C1[int]).split('.')[-1], 'C1[int]') self.assertEqual(C2.__parameters__, ()) - self.assertIsInstance(C2(), collections_abc.Callable) - self.assertIsSubclass(C2, collections_abc.Callable) - self.assertIsSubclass(C1, collections_abc.Callable) + self.assertIsInstance(C2(), collections.abc.Callable) + self.assertIsSubclass(C2, collections.abc.Callable) + self.assertIsSubclass(C1, collections.abc.Callable) self.assertIsInstance(T1(), tuple) self.assertIsSubclass(T2, tuple) self.assertIsSubclass(Tuple[int, ...], typing.Sequence) @@ -1020,21 +1012,6 @@ def test_all_repr_eq_any(self): self.assertNotEqual(repr(base), '') self.assertEqual(base, base) - def test_substitution_helper(self): - T = TypeVar('T') - KT = TypeVar('KT') - VT = TypeVar('VT') - class Map(Generic[KT, VT]): - def meth(self, k: KT, v: VT): ... - StrMap = Map[str, T] - obj = StrMap[int]() - - new_args = typing._subs_tree(obj.__orig_class__) - new_annots = {k: typing._replace_arg(v, type(obj).__parameters__, new_args) - for k, v in obj.meth.__annotations__.items()} - - self.assertEqual(new_annots, {'k': str, 'v': int}) - def test_pickle(self): global C # pickle wants to reference the class by name T = TypeVar('T') @@ -1584,7 +1561,7 @@ async def __aexit__(self, etype, eval, tb): PY36_TESTS = """ from test import ann_module, ann_module2, ann_module3 -from typing import AsyncContextManager +#from typing import AsyncContextManager class A: y: float @@ -1626,15 +1603,15 @@ class HasForeignBaseClass(mod_generics_cache.A): some_xrepr: 'XRepr' other_a: 'mod_generics_cache.A' -async def g_with(am: AsyncContextManager[int]): - x: int - async with am as x: - return x - -try: - g_with(ACM()).send(None) -except StopIteration as e: - assert e.args[0] == 42 +#async def g_with(am: AsyncContextManager[int]): +# x: int +# async with am as x: +# return x +# +#try: +# g_with(ACM()).send(None) +#except StopIteration as e: +# assert e.args[0] == 42 """ if PY36: @@ -2101,11 +2078,11 @@ def __len__(self): self.assertIsSubclass(MMC, typing.Mapping) self.assertIsInstance(MMB[KT, VT](), typing.Mapping) - self.assertIsInstance(MMB[KT, VT](), collections_abc.Mapping) + self.assertIsInstance(MMB[KT, VT](), collections.abc.Mapping) - self.assertIsSubclass(MMA, collections_abc.Mapping) - self.assertIsSubclass(MMB, collections_abc.Mapping) - self.assertIsSubclass(MMC, collections_abc.Mapping) + self.assertIsSubclass(MMA, collections.abc.Mapping) + self.assertIsSubclass(MMB, collections.abc.Mapping) + self.assertIsSubclass(MMC, collections.abc.Mapping) self.assertIsSubclass(MMB[str, str], typing.Mapping) self.assertIsSubclass(MMC, MMA) @@ -2117,9 +2094,8 @@ class G(typing.Generator[int, int, int]): ... def g(): yield 0 self.assertIsSubclass(G, typing.Generator) self.assertIsSubclass(G, typing.Iterable) - if hasattr(collections_abc, 'Generator'): - self.assertIsSubclass(G, collections_abc.Generator) - self.assertIsSubclass(G, collections_abc.Iterable) + self.assertIsSubclass(G, collections.abc.Generator) + self.assertIsSubclass(G, collections.abc.Iterable) self.assertNotIsSubclass(type(g), G) @skipUnless(PY36, 'Python 3.6 required') @@ -2135,15 +2111,15 @@ def athrow(self, typ, val=None, tb=None): g = ns['g'] self.assertIsSubclass(G, typing.AsyncGenerator) self.assertIsSubclass(G, typing.AsyncIterable) - self.assertIsSubclass(G, collections_abc.AsyncGenerator) - self.assertIsSubclass(G, collections_abc.AsyncIterable) + self.assertIsSubclass(G, collections.abc.AsyncGenerator) + self.assertIsSubclass(G, collections.abc.AsyncIterable) self.assertNotIsSubclass(type(g), G) instance = G() self.assertIsInstance(instance, typing.AsyncGenerator) self.assertIsInstance(instance, typing.AsyncIterable) - self.assertIsInstance(instance, collections_abc.AsyncGenerator) - self.assertIsInstance(instance, collections_abc.AsyncIterable) + self.assertIsInstance(instance, collections.abc.AsyncGenerator) + self.assertIsInstance(instance, collections.abc.AsyncIterable) self.assertNotIsInstance(type(g), G) self.assertNotIsInstance(g, G) @@ -2180,23 +2156,23 @@ class D: ... self.assertIsSubclass(D, B) class M(): ... - collections_abc.MutableMapping.register(M) + collections.abc.MutableMapping.register(M) self.assertIsSubclass(M, typing.Mapping) def test_collections_as_base(self): - class M(collections_abc.Mapping): ... + class M(collections.abc.Mapping): ... self.assertIsSubclass(M, typing.Mapping) self.assertIsSubclass(M, typing.Iterable) - class S(collections_abc.MutableSequence): ... + class S(collections.abc.MutableSequence): ... self.assertIsSubclass(S, typing.MutableSequence) self.assertIsSubclass(S, typing.Iterable) - class I(collections_abc.Iterable): ... + class I(collections.abc.Iterable): ... self.assertIsSubclass(I, typing.Iterable) - class A(collections_abc.Mapping, metaclass=abc.ABCMeta): ... + class A(collections.abc.Mapping, metaclass=abc.ABCMeta): ... class B: ... A.register(B) self.assertIsSubclass(B, typing.Mapping) @@ -2213,7 +2189,7 @@ def manager(): self.assertIsInstance(cm, typing.ContextManager) self.assertNotIsInstance(42, typing.ContextManager) - @skipUnless(ASYNCIO, 'Python 3.5 required') + @skipUnless(False, "Temporary") # (ASYNCIO, 'Python 3.5 required') def test_async_contextmanager(self): class NotACM: pass diff --git a/Lib/typing.py b/Lib/typing.py index 1168f3c555cd04..1d724f2b8d9da1 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -1,12 +1,12 @@ import abc from abc import abstractmethod, abstractproperty import collections +import collections.abc import contextlib import functools import re as stdlib_re # Avoid confusion with the re we export. import sys import types -import collections.abc as collections_abc from types import WrapperDescriptorType, MethodWrapperType, MethodDescriptorType # Please keep __all__ alphabetized within each category. @@ -24,12 +24,10 @@ # ABCs (from collections.abc). 'AbstractSet', # collections.abc.Set. - 'GenericMeta', # subclass of abc.ABCMeta and a metaclass - # for 'Generic' and ABCs below. - 'ByteString', + 'ByteString', # collections.abc.ByteString. 'Container', 'ContextManager', - 'Hashable', + 'Hashable', # collections.abc.Hashable. 'ItemsView', 'Iterable', 'Iterator', @@ -48,7 +46,7 @@ 'Coroutine', 'Collection', 'AsyncGenerator', - 'AsyncContextManager', + # 'AsyncContextManager', # Structural checks, a.k.a. protocols. 'Reversible', @@ -156,7 +154,7 @@ def _type_repr(obj): typically enough to uniquely identify a type. For everything else, we fall back on repr(obj). """ - if isinstance(obj, type) and not issubclass(obj, _TypingBase): + if isinstance(obj, type): if obj.__module__ == 'builtins': return obj.__qualname__ return '%s.%s' % (obj.__module__, obj.__qualname__) @@ -267,7 +265,7 @@ def _get_type_vars(self, tvars): def __repr__(self): cls = type(self) - qname = _trim_name(_qualname(cls)) + qname = _trim_name(cls.__qualname__) return '%s.%s' % (cls.__module__, qname) def __call__(self, *args, **kwds): @@ -317,7 +315,6 @@ class _ForwardRef(_FinalTypingBase, _root=True): '__forward_evaluated__', '__forward_value__') def __init__(self, arg): - super().__init__(arg) if not isinstance(arg, str): raise TypeError('Forward reference must be a string -- got %r' % (arg,)) try: @@ -611,22 +608,20 @@ def __init__(self, origin, params): keyword arguments that are used for internal bookkeeping, therefore an override should pass unused keyword arguments to super(). """ + if not isinstance(params, tuple): + params = (params,) self.__origin__ = origin self.__args__ = params self.__parameters__ = _type_vars(params) def _get_type_vars(self, tvars): - if self.__origin__ and self.__parameters__: - _get_type_vars(self.__parameters__, tvars) + _get_type_vars(self.__parameters__, tvars) def _eval_type(self, globalns, localns): - ev_origin = (self.__origin__._eval_type(globalns, localns) - if self.__origin__ else None) - ev_args = tuple(_eval_type(a, globalns, localns) for a - in self.__args__) if self.__args__ else None - if ev_origin == self.__origin__ and ev_args == self.__args__: + ev_args = tuple(_eval_type(a, globalns, localns) for a in self.__args__) + if ev_args == self.__args__: return self - return _GenericAlias() + return _GenericAlias(self.__origin__, ev_args) @_tp_cache def __getitem__(self, parameters): @@ -636,14 +631,74 @@ def __getitem__(self, parameters): repr(self)) def __repr__(self): - if self.__origin__ is None: - return super().__repr__() - return self._tree_repr(self._subs_tree()) + return f'{_type_repr(self.__origin__)}[{", ".join([_type_repr(a) for a in self.__args__])}]' + + def __eq__(self, other): + if not isinstance(other, _GenericAlias): + return NotImplemented + if self.__origin__ != other.__origin__: + return False + if self.__origin__ is Union and other.__origin__ is Union: + return frozenset(self.__args__) == frozenset(other.__args__) + return self.__args__ == other.__args__ + + def __hash__(self): + if self.__origin__ is Union: + return hash((Union, frozenset(self.__args__))) + return hash((self.__origin__, self.__args__)) + + def __call__(self, *args, **kwargs): + result = self.__origin__(*args, **kwargs) + try: + result.__orig_class__ = self + except AttributeError: + pass + return result def __mro_entry__(self, bases): return self.__origin__ - # TODO: __getattr__ and __setattr__. + def __getitem__(self, params): + return _GenericAlias(self.__origin__, params) + + def __getattr__(self, attr): + if '__origin__' in self.__dict__: # We are carefull for copy and pickle + return getattr(self.__origin__, attr) + raise AttributeError(attr) + + # TODO: __setattr__. + + + +def _make_subclasshook(cls): + """Construct a __subclasshook__ callable that incorporates + the associated __extra__ class in subclass checks performed + against cls. + """ + extra = cls.__bases__[0] + if isinstance(extra, abc.ABCMeta): + # The logic mirrors that of ABCMeta.__subclasscheck__. + # Registered classes need not be checked here because + # cls and its extra share the same _abc_registry. + def __extrahook__(subclass): + res = extra.__subclasshook__(subclass) + if res is not NotImplemented: + return res + if extra in subclass.__mro__: + return True + for scls in extra.__subclasses__(): + if scls is not cls and issubclass(subclass, scls): + return True + if subclass in extra._abc_registry: + return True + return NotImplemented + else: + # For non-ABC extras we'll just call issubclass(). + def __extrahook__(subclass): + if issubclass(subclass, extra): + return True + return NotImplemented + return __extrahook__ class Generic(_TypingBase): @@ -713,6 +768,8 @@ def __init_subclass__(cls, *args, **kwargs): if hasattr(cls, '__orig_bases__'): pars = _type_vars(cls.__orig_bases__) cls.__parameters__ = tuple(pars) + if cls.__module__ == 'typing': + cls.__subclasshook__ = _make_subclasshook(cls) class _TypingEmpty: @@ -745,22 +802,22 @@ def __new__(cls, *args, **kwds): return super().__new__(cls, *args, **kwds) @_tp_cache - def __class_getitem__(self, parameters): - if self.__origin__ is not None or self._gorg is not Tuple: + def __class_getitem__(cls, parameters): + if cls is not Tuple: # Normal generic rules apply if this is not the first subscription # or a subscription of a subclass. - return super().__getitem__(parameters) + return super().__class_getitem__(cls, parameters) if parameters == (): - return super().__getitem__((_TypingEmpty,)) + return super().__class_getitem__(cls, (_TypingEmpty,)) if not isinstance(parameters, tuple): parameters = (parameters,) if len(parameters) == 2 and parameters[1] is ...: msg = "Tuple[t, ...]: t must be a type." p = _type_check(parameters[0], msg) - return super().__getitem__((p, _TypingEllipsis)) + return super().__class_getitem__(cls, (p, _TypingEllipsis)) msg = "Tuple[t0, t1, ...]: each t must be a type." parameters = tuple(_type_check(p, msg) for p in parameters) - return super().__getitem__(parameters) + return super().__class_getitem__(cls, parameters) class Callable(collections.abc.Callable, Generic): @@ -782,13 +839,13 @@ def __new__(cls, *args, **kwds): "use a non-abstract subclass instead") return super().__new__(cls, *args, **kwds) - def __class_getitem__(self, parameters): + def __class_getitem__(cls, parameters): """A thin wrapper around __getitem_inner__ to provide the latter with hashable arguments to improve speed. """ - if self.__origin__ is not None or self._gorg is not Callable: - return super().__getitem__(parameters) + if cls is not Callable: + return super().__class_getitem__(cls, parameters) if not isinstance(parameters, tuple) or len(parameters) != 2: raise TypeError("Callable must be used as " "Callable[[arg, ...], result].") @@ -800,19 +857,20 @@ def __class_getitem__(self, parameters): raise TypeError("Callable[args, result]: args must be a list." " Got %.100r." % (args,)) parameters = (tuple(args), result) - return self.__getitem_inner__(parameters) + return cls.__getitem_inner__(parameters) + @classmethod @_tp_cache - def __getitem_inner__(self, parameters): + def __getitem_inner__(cls, parameters): args, result = parameters msg = "Callable[args, result]: result must be a type." result = _type_check(result, msg) if args is Ellipsis: - return super().__getitem__((_TypingEllipsis, result)) + return super().__class_getitem__(cls, (_TypingEllipsis, result)) msg = "Callable[[arg, ...], result]: each arg must be a type." args = tuple(_type_check(arg, msg) for arg in args) parameters = args + (result,) - return super().__getitem__(parameters) + return super().__class_getitem__(cls, parameters) def cast(typ, val): From e764c6121a18665996917ee428dd8e8bb434c984 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 12 Nov 2017 17:47:08 +0100 Subject: [PATCH 29/82] More fixes --- Lib/typing.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/Lib/typing.py b/Lib/typing.py index 1d724f2b8d9da1..94fb1bd8423bba 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -666,7 +666,10 @@ def __getattr__(self, attr): return getattr(self.__origin__, attr) raise AttributeError(attr) - # TODO: __setattr__. + def __setattr__(self, attr, val): + if attr not in ('__origin__', '__args__', '__parameters__'): + setattr(self.__origin__, attr, val) + self.__dict__[attr] = val @@ -675,6 +678,10 @@ def _make_subclasshook(cls): the associated __extra__ class in subclass checks performed against cls. """ + if cls.__module__ != 'typing': + def __extrahook__(subclass): + return NotImplemented + return __extrahook__ extra = cls.__bases__[0] if isinstance(extra, abc.ABCMeta): # The logic mirrors that of ABCMeta.__subclasscheck__. @@ -686,11 +693,12 @@ def __extrahook__(subclass): return res if extra in subclass.__mro__: return True + for rcls in extra._abc_registry: + if rcls is not cls and issubclass(subclass, rcls): + return True for scls in extra.__subclasses__(): if scls is not cls and issubclass(subclass, scls): return True - if subclass in extra._abc_registry: - return True return NotImplemented else: # For non-ABC extras we'll just call issubclass(). @@ -721,7 +729,6 @@ def lookup_name(mapping: Mapping[KT, VT], key: KT, default: VT) -> VT: except KeyError: return default """ - __slots__ = () def __new__(cls, *args, **kwds): @@ -768,7 +775,7 @@ def __init_subclass__(cls, *args, **kwargs): if hasattr(cls, '__orig_bases__'): pars = _type_vars(cls.__orig_bases__) cls.__parameters__ = tuple(pars) - if cls.__module__ == 'typing': + if '__subclasshook__' not in cls.__dict__ or self.__subclasshook__.__name__ == '__extrahook__': cls.__subclasshook__ = _make_subclasshook(cls) @@ -783,7 +790,7 @@ class _TypingEllipsis: """Internal placeholder for ... (ellipsis).""" -class Tuple(tuple, Generic): +class Tuple(tuple, Generic, metaclass=abc.ABCMeta): """Tuple type; Tuple[X, Y] is the cross-product type of X and Y. Example: Tuple[T1, T2] is a tuple of two elements corresponding @@ -1278,7 +1285,7 @@ class MutableSequence(collections.abc.MutableSequence, Generic[T]): ByteString = collections.abc.ByteString -class List(list, Generic[T]): +class List(list, Generic[T], metaclass=abc.ABCMeta): __slots__ = () def __new__(cls, *args, **kwds): @@ -1292,7 +1299,7 @@ class Deque(collections.deque, Generic[T]): __slots__ = () -class Set(set, Generic[T]): +class Set(set, Generic[T], metaclass=abc.ABCMeta): __slots__ = () def __new__(cls, *args, **kwds): @@ -1302,7 +1309,7 @@ def __new__(cls, *args, **kwds): return super().__new__(cls, *args, **kwds) -class FrozenSet(frozenset, Generic[T_co]): +class FrozenSet(frozenset, Generic[T_co], metaclass=abc.ABCMeta): __slots__ = () def __new__(cls, *args, **kwds): @@ -1336,7 +1343,7 @@ class ContextManager(contextlib.AbstractContextManager, Generic[T_co]): # __slots__ = () -class Dict(dict, Generic[KT, VT]): +class Dict(dict, Generic[KT, VT], metaclass=abc.ABCMeta): __slots__ = () def __new__(cls, *args, **kwds): From c75d80c743ae508c907379d99ae76133d879f7b2 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 12 Nov 2017 18:19:06 +0100 Subject: [PATCH 30/82] Return old _TypeAlias; more fixes --- Lib/typing.py | 116 ++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 107 insertions(+), 9 deletions(-) diff --git a/Lib/typing.py b/Lib/typing.py index 94fb1bd8423bba..bde1f0bca997cc 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -97,7 +97,7 @@ def _trim_name(nm): def _get_type_vars(types, tvars): for t in types: - if isinstance(t, type) and issubclass(t, _TypingBase) or isinstance(t, _TypingBase): + if isinstance(t, _TypingBase): t._get_type_vars(tvars) @@ -108,7 +108,7 @@ def _type_vars(types): def _eval_type(t, globalns, localns): - if isinstance(t, type) and issubclass(t, _TypingBase) or isinstance(t, _TypingBase): + if isinstance(t, _TypingBase): return t._eval_type(globalns, localns) return t @@ -341,6 +341,15 @@ def _eval_type(self, globalns, localns): self.__forward_evaluated__ = True return self.__forward_value__ + def __eq__(self, other): + if not isinstance(other, _ForwardRef): + return NotImplemented + return (self.__forward_arg__ == other.__forward_arg__ and + self.__forward_value__ == other.__forward_value__) + + def __hash__(self): + return hash((self.__forward_arg__, self.__forward_value__)) + def __repr__(self): return '_ForwardRef(%r)' % (self.__forward_arg__,) @@ -775,7 +784,7 @@ def __init_subclass__(cls, *args, **kwargs): if hasattr(cls, '__orig_bases__'): pars = _type_vars(cls.__orig_bases__) cls.__parameters__ = tuple(pars) - if '__subclasshook__' not in cls.__dict__ or self.__subclasshook__.__name__ == '__extrahook__': + if cls.__module__ == 'typing' or cls.__subclasshook__.__name__ == '__extrahook__': cls.__subclasshook__ = _make_subclasshook(cls) @@ -1295,9 +1304,15 @@ def __new__(cls, *args, **kwds): return super().__new__(cls, *args, **kwds) -class Deque(collections.deque, Generic[T]): +class Deque(collections.deque, Generic[T], metaclass=abc.ABCMeta): __slots__ = () + def __new__(cls, *args, **kwargs): + if cls is Deque: + return collections.deque(*args, **kwargs) + return super().__new__(cls, *args, **kwargs) + + class Set(set, Generic[T], metaclass=abc.ABCMeta): __slots__ = () @@ -1353,17 +1368,32 @@ def __new__(cls, *args, **kwds): return super().__new__(cls, *args, **kwds) -class DefaultDict(collections.defaultdict, Generic[KT, VT]): +class DefaultDict(collections.defaultdict, Generic[KT, VT], metaclass=abc.ABCMeta): __slots__ = () + def __new__(cls, *args, **kwargs): + if cls is DefaultDict: + return collections.defaultdict(*args, **kwargs) + return super().__new__(cls, *args, **kwargs) + -class Counter(collections.Counter, Generic[T]): +class Counter(collections.Counter, Generic[T], metaclass=abc.ABCMeta): __slots__ = () + def __new__(cls, *args, **kwargs): + if cls is Counter: + return collections.Counter(*args, **kwargs) + return super().__new__(cls, *args, **kwargs) -class ChainMap(collections.ChainMap, Generic[KT, VT]): + +class ChainMap(collections.ChainMap, Generic[KT, VT], metaclass=abc.ABCMeta): __slots__ = () + def __new__(cls, *args, **kwargs): + if cls is ChainMap: + return collections.ChainMap(*args, **kwargs) + return super().__new__(cls, *args, **kwargs) + class Generator(collections.abc.Generator, Generic[T_co, T_contra, V_co]): __slots__ = () @@ -1695,9 +1725,77 @@ class io: io.__name__ = __name__ + '.io' sys.modules[io.__name__] = io -Pattern = _GenericAlias(type(stdlib_re.compile('')), (AnyStr,)) -Match = _GenericAlias(type(stdlib_re.match('', '')), (AnyStr,)) +class _TypeAlias(_FinalTypingBase, _root=True): + """Internal helper class for defining generic variants of concrete types. + + Note that this is not a type; let's call it a pseudo-type. It cannot + be used in instance and subclass checks in parameterized form, i.e. + ``isinstance(42, Match[str])`` raises ``TypeError`` instead of returning + ``False``. + """ + + __slots__ = ('name', 'type_var', 'impl_type', 'type_checker') + + def __init__(self, name, type_var, impl_type, type_checker): + """Initializer. + + Args: + name: The name, e.g. 'Pattern'. + type_var: The type parameter, e.g. AnyStr, or the + specific type, e.g. str. + impl_type: The implementation type. + type_checker: Function that takes an impl_type instance. + and returns a value that should be a type_var instance. + """ + assert isinstance(name, str), repr(name) + assert isinstance(impl_type, type), repr(impl_type) + assert isinstance(type_var, (type, _TypingBase)), repr(type_var) + self.name = name + self.type_var = type_var + self.impl_type = impl_type + self.type_checker = type_checker + + def __repr__(self): + return "%s[%s]" % (self.name, _type_repr(self.type_var)) + + def __getitem__(self, parameter): + if not isinstance(self.type_var, TypeVar): + raise TypeError("%s cannot be further parameterized." % self) + if self.type_var.__constraints__ and isinstance(parameter, type): + if not issubclass(parameter, self.type_var.__constraints__): + raise TypeError("%s is not a valid substitution for %s." % + (parameter, self.type_var)) + if isinstance(parameter, TypeVar) and parameter is not self.type_var: + raise TypeError("%s cannot be re-parameterized." % self) + return self.__class__(self.name, parameter, + self.impl_type, self.type_checker) + + def __eq__(self, other): + if not isinstance(other, _TypeAlias): + return NotImplemented + return self.name == other.name and self.type_var == other.type_var + + def __hash__(self): + return hash((self.name, self.type_var)) + + def __instancecheck__(self, obj): + if not isinstance(self.type_var, TypeVar): + raise TypeError("Parameterized type aliases cannot be used " + "with isinstance().") + return isinstance(obj, self.impl_type) + + def __subclasscheck__(self, cls): + if not isinstance(self.type_var, TypeVar): + raise TypeError("Parameterized type aliases cannot be used " + "with issubclass().") + return issubclass(cls, self.impl_type) + + +Pattern = _TypeAlias('Pattern', AnyStr, type(stdlib_re.compile('')), + lambda p: p.pattern) +Match = _TypeAlias('Match', AnyStr, type(stdlib_re.match('', '')), + lambda m: m.re.pattern) class re: """Wrapper namespace for re type aliases.""" From 294872449aeade0b02a48b343f98cd0d55437c47 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 12 Nov 2017 18:30:48 +0100 Subject: [PATCH 31/82] Return old _TypeAlias; more fixes --- Lib/typing.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/Lib/typing.py b/Lib/typing.py index bde1f0bca997cc..6d8439e85e8de4 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -165,6 +165,18 @@ def _type_repr(obj): return repr(obj) +def _subs_tvars(tp, tvars, subs): + assert isinstance(tp, _GenericAlias) + new_args = list(tp.__args__) + for a, arg in enumerate(tp.__args__): + if isinstance(arg, TypeVar): + for i, tvar in enumerate(tvars): + if arg == tvar: + new_args[a] = subs + new_args[a] = _subs_tvars(arg, tvars, subs) + return _GenericAlias(tp.__origin__, tuple(new_args)) + + def _remove_dups_flatten(parameters): """An internal helper for Union creation and substitution: flatten Union's among parameters, then remove duplicates and strict subclasses. @@ -668,7 +680,12 @@ def __mro_entry__(self, bases): return self.__origin__ def __getitem__(self, params): - return _GenericAlias(self.__origin__, params) + if not isinstamce(params, tuple): + params = (params,) + msg = "Parameters to generic types must be types." + params = tuple(_type_check(p, msg) for p in params) + _check_generic(self, params) + return _subs_tvars(self, self.__parameters__, params) def __getattr__(self, attr): if '__origin__' in self.__dict__: # We are carefull for copy and pickle From 4b8d8f3c7e91a7982d375aed09ab0017f5f5a501 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 12 Nov 2017 20:35:16 +0100 Subject: [PATCH 32/82] Some more fixes --- Lib/typing.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/Lib/typing.py b/Lib/typing.py index 6d8439e85e8de4..420847777fe9da 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -166,14 +166,18 @@ def _type_repr(obj): def _subs_tvars(tp, tvars, subs): - assert isinstance(tp, _GenericAlias) + if not isinstance(tp, _GenericAlias): + return tp new_args = list(tp.__args__) for a, arg in enumerate(tp.__args__): if isinstance(arg, TypeVar): for i, tvar in enumerate(tvars): if arg == tvar: - new_args[a] = subs - new_args[a] = _subs_tvars(arg, tvars, subs) + new_args[a] = subs[i] + else: + new_args[a] = _subs_tvars(arg, tvars, subs) + if tp.__origin__ is Union: + return Union[tuple(new_args)] return _GenericAlias(tp.__origin__, tuple(new_args)) @@ -286,6 +290,8 @@ def __call__(self, *args, **kwds): class _FinalTypingBase(_TypingBase): + __slots__ = () + def __init_subclass__(self, *args, **kwds): if not kwds.pop('_root', False): raise TypeError("Cannot subclass special typing classes") @@ -624,6 +630,7 @@ def __getitem__(self, arg): class _GenericAlias(_FinalTypingBase, _root=True): + def __init__(self, origin, params): """Create a new generic class. GenericMeta.__new__ accepts keyword arguments that are used for internal bookkeeping, therefore @@ -677,10 +684,15 @@ def __call__(self, *args, **kwargs): return result def __mro_entry__(self, bases): + if self.__origin__ is Generic: + i = bases.index(self) + for b in bases[i+1:]: + if isinstance(b, _GenericAlias): + return None return self.__origin__ def __getitem__(self, params): - if not isinstamce(params, tuple): + if not isinstance(params, tuple): params = (params,) msg = "Parameters to generic types must be types." params = tuple(_type_check(p, msg) for p in params) @@ -711,8 +723,6 @@ def __extrahook__(subclass): extra = cls.__bases__[0] if isinstance(extra, abc.ABCMeta): # The logic mirrors that of ABCMeta.__subclasscheck__. - # Registered classes need not be checked here because - # cls and its extra share the same _abc_registry. def __extrahook__(subclass): res = extra.__subclasshook__(subclass) if res is not NotImplemented: @@ -827,6 +837,7 @@ class Tuple(tuple, Generic, metaclass=abc.ABCMeta): """ __slots__ = () + __call__ = None def __new__(cls, *args, **kwds): if cls is Tuple: From 85e1f58209a7ddef884f02b742a37a38d3f8fd7b Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 12 Nov 2017 21:55:28 +0100 Subject: [PATCH 33/82] Even more fixes; use f-strings --- Lib/test/test_typing.py | 28 ++++------ Lib/typing.py | 116 ++++++++++++++++++++++++---------------- 2 files changed, 82 insertions(+), 62 deletions(-) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 2a3d311a0eec43..1fb01bbc3d11a4 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -399,7 +399,7 @@ def test_tuple_instance_type_error(self): self.assertIsInstance((0, 0), Tuple) def test_repr(self): - self.assertEqual(repr(Tuple), 'typing.Tuple') + self.assertEqual(repr(Tuple), "") self.assertEqual(repr(Tuple[()]), 'typing.Tuple[()]') self.assertEqual(repr(Tuple[int, float]), 'typing.Tuple[int, float]') self.assertEqual(repr(Tuple[int, ...]), 'typing.Tuple[int, ...]') @@ -626,9 +626,9 @@ def test_init(self): def test_repr(self): self.assertEqual(repr(SimpleMapping), - __name__ + '.' + 'SimpleMapping') + "") self.assertEqual(repr(MySimpleMapping), - __name__ + '.' + 'MySimpleMapping') + "") def test_chain_repr(self): T = TypeVar('T') @@ -659,7 +659,7 @@ def test_new_repr(self): U = TypeVar('U', covariant=True) S = TypeVar('S') - self.assertEqual(repr(List), 'typing.List') + self.assertEqual(repr(List), "") self.assertEqual(repr(List[T]), 'typing.List[~T]') self.assertEqual(repr(List[U]), 'typing.List[+U]') self.assertEqual(repr(List[S][T][int]), 'typing.List[int]') @@ -907,7 +907,7 @@ def test_extended_generic_rules_repr(self): self.assertEqual(repr(Union[Tuple, Callable]).replace('typing.', ''), 'Union[Tuple, Callable]') self.assertEqual(repr(Union[Tuple, Tuple[int]]).replace('typing.', ''), - 'Tuple') + "") self.assertEqual(repr(Callable[..., Optional[T]][int]).replace('typing.', ''), 'Callable[..., Union[int, NoneType]]') self.assertEqual(repr(Callable[[], List[T]][int]).replace('typing.', ''), @@ -1111,29 +1111,23 @@ class C(Generic[B]): pass def test_repr_2(self): - PY32 = sys.version_info[:2] < (3, 3) - class C(Generic[T]): pass self.assertEqual(C.__module__, __name__) - if not PY32: - self.assertEqual(C.__qualname__, - 'GenericTests.test_repr_2..C') - self.assertEqual(repr(C).split('.')[-1], 'C') + self.assertEqual(C.__qualname__, + 'GenericTests.test_repr_2..C') X = C[int] self.assertEqual(X.__module__, __name__) - if not PY32: - self.assertTrue(X.__qualname__.endswith('..C')) + self.assertTrue(X.__qualname__.endswith('..C')) self.assertEqual(repr(X).split('.')[-1], 'C[int]') class Y(C[int]): pass self.assertEqual(Y.__module__, __name__) - if not PY32: - self.assertEqual(Y.__qualname__, - 'GenericTests.test_repr_2..Y') + self.assertEqual(Y.__qualname__, + 'GenericTests.test_repr_2..Y') self.assertEqual(repr(Y).split('.')[-1], 'Y') def test_eq_1(self): @@ -2475,7 +2469,7 @@ class A(typing.Match): pass self.assertEqual(str(ex.exception), - "Cannot subclass typing._TypeAlias") + "Cannot subclass ") class AllTests(BaseTestCase): diff --git a/Lib/typing.py b/Lib/typing.py index 420847777fe9da..622819113f403f 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -114,7 +114,12 @@ def _eval_type(t, globalns, localns): Generic = object() +_GenericAlias = None _Protocol = object() +ClassVar = object() +Union = object() +NoReturn = object() +Optional = object() def _type_check(arg, msg): """Check that the argument is a type, and return it (internal helper). @@ -131,18 +136,15 @@ def _type_check(arg, msg): if arg is None: return type(None) if isinstance(arg, str): - arg = _ForwardRef(arg) + return _ForwardRef(arg) if ( - isinstance(arg, _TypingBase) and type(arg).__name__ == '_ClassVar' or - not isinstance(arg, (type, _TypingBase)) and not callable(arg) - ): - raise TypeError(msg + " Got %.100r." % (arg,)) - # Bare Union etc. are not valid as type arguments - if ( - type(arg).__name__ in ('_Union', '_Optional') and - not getattr(arg, '__origin__', None) or arg in (Generic, _Protocol) + # Bare Union etc. are not valid as type arguments + _GenericAlias and isinstance(arg, _GenericAlias) and arg.__origin__ in (Generic, _Protocol, ClassVar) or + arg in (Generic, _Protocol, ClassVar, Union, NoReturn, Optional) ): raise TypeError("Plain %s is not valid as type argument" % arg) + if not callable(arg): + raise TypeError(msg + " Got %.100r." % (arg,)) return arg @@ -212,7 +214,7 @@ def _remove_dups_flatten(parameters): # (In particular, Union[str, AnyStr] != AnyStr.) all_params = set(params) for t1 in params: - if not isinstance(t1, type): + if not isinstance(t1, (type, _GenericAlias)): continue if any(isinstance(t2, type) and issubclass(t1, t2) for t2 in all_params - {t1}): @@ -639,7 +641,9 @@ def __init__(self, origin, params): if not isinstance(params, tuple): params = (params,) self.__origin__ = origin - self.__args__ = params + self.__args__ = tuple(... if a is _TypingEllipsis else + () if a is _TypingEmpty else + a for a in params) self.__parameters__ = _type_vars(params) def _get_type_vars(self, tvars): @@ -652,14 +656,25 @@ def _eval_type(self, globalns, localns): return _GenericAlias(self.__origin__, ev_args) @_tp_cache - def __getitem__(self, parameters): + def __getitem__(self, params): if self.__origin__ in (Generic, _Protocol): # Can't subscript Generic[...] or _Protocol[...]. - raise TypeError("Cannot subscript already-subscripted %s" % - repr(self)) + raise TypeError("Cannot subscript already-subscripted {self}") + if not isinstance(params, tuple): + params = (params,) + msg = "Parameters to generic types must be types." + params = tuple(_type_check(p, msg) for p in params) + _check_generic(self, params) + return _subs_tvars(self, self.__parameters__, params) def __repr__(self): - return f'{_type_repr(self.__origin__)}[{", ".join([_type_repr(a) for a in self.__args__])}]' + if (self.__origin__ is not Callable or + len(self.__args__) == 2 and self.__args__[0] is Ellipsis): + return (f'{_type_repr(self.__origin__)}' + f'[{", ".join([_type_repr(a) for a in self.__args__])}]') + return (f'typing.Callable' + f'[[{", ".join([_type_repr(a) for a in self.__args__[:-1]])}], ' + f'{_type_repr(self.__args__[-1])}]') def __eq__(self, other): if not isinstance(other, _GenericAlias): @@ -691,14 +706,6 @@ def __mro_entry__(self, bases): return None return self.__origin__ - def __getitem__(self, params): - if not isinstance(params, tuple): - params = (params,) - msg = "Parameters to generic types must be types." - params = tuple(_type_check(p, msg) for p in params) - _check_generic(self, params) - return _subs_tvars(self, self.__parameters__, params) - def __getattr__(self, attr): if '__origin__' in self.__dict__: # We are carefull for copy and pickle return getattr(self.__origin__, attr) @@ -710,7 +717,6 @@ def __setattr__(self, attr, val): self.__dict__[attr] = val - def _make_subclasshook(cls): """Construct a __subclasshook__ callable that incorporates the associated __extra__ class in subclass checks performed @@ -745,7 +751,7 @@ def __extrahook__(subclass): return __extrahook__ -class Generic(_TypingBase): +class Generic: """Abstract base class for generic types. A generic type is typically declared by inheriting from @@ -771,7 +777,7 @@ def __new__(cls, *args, **kwds): if cls is Generic: raise TypeError("Type Generic cannot be instantiated; " "it can be used only as a base class") - return super().__new__(cls, *args, **kwds) + return super().__new__(cls) @_tp_cache def __class_getitem__(cls, params): @@ -779,7 +785,7 @@ def __class_getitem__(cls, params): params = (params,) if not params and cls is not Tuple: raise TypeError( - "Parameter list to %s[...] cannot be empty" % cls.__qualname__) + f"Parameter list to {cls.__qualname__}[...] cannot be empty") msg = "Parameters to generic types must be types." params = tuple(_type_check(p, msg) for p in params) if cls is Generic: @@ -790,27 +796,49 @@ def __class_getitem__(cls, params): if len(set(params)) != len(params): raise TypeError( "Parameters to Generic[...] must all be unique") - tvars = params - args = params elif cls in (Tuple, Callable): - tvars = _type_vars(params) - args = params + pass elif cls is _Protocol: # _Protocol is internal, don't check anything. - tvars = params - args = params + pass else: # Subscripting a regular Generic subclass. _check_generic(cls, params) - tvars = _type_vars(params) - args = params return _GenericAlias(cls, params) def __init_subclass__(cls, *args, **kwargs): - pars = [] + tvars = [] + if (not hasattr(cls, '__orig_bases__') and Generic in cls.__bases__ and + cls.__name__ not in ('Tuple', 'Callable') or + hasattr(cls, '__orig_bases__') and Generic in cls.__orig_bases__): + raise TypeError("Cannot inherit from plain Generic") if hasattr(cls, '__orig_bases__'): - pars = _type_vars(cls.__orig_bases__) - cls.__parameters__ = tuple(pars) + tvars = _type_vars(cls.__orig_bases__) + # Look for Generic[T1, ..., Tn]. + # If found, tvars must be a subset of it. + # If not found, tvars is it. + # Also check for and reject plain Generic, + # and reject multiple Generic[...]. + gvars = None + for base in cls.__orig_bases__: + if (isinstance(base, _GenericAlias) and + base.__origin__ is Generic): + if gvars is not None: + raise TypeError( + "Cannot inherit from Generic[...] multiple types.") + gvars = base.__parameters__ + if gvars is None: + gvars = tvars + else: + tvarset = set(tvars) + gvarset = set(gvars) + if not tvarset <= gvarset: + raise TypeError( + f"Some type variables " + "({', '.join(str(t) for t in tvars if t not in gvarset)}) " + "are not listed in Generic[{', '.join(str(g) for g in gvars)}]") + tvars = gvars + cls.__parameters__ = tuple(tvars) if cls.__module__ == 'typing' or cls.__subclasshook__.__name__ == '__extrahook__': cls.__subclasshook__ = _make_subclasshook(cls) @@ -898,8 +926,7 @@ def __class_getitem__(cls, parameters): parameters = (Ellipsis, result) else: if not isinstance(args, list): - raise TypeError("Callable[args, result]: args must be a list." - " Got %.100r." % (args,)) + raise TypeError(f"Callable[args, result]: args must be a list. Got {args}") parameters = (tuple(args), result) return cls.__getitem_inner__(parameters) @@ -1785,17 +1812,16 @@ def __init__(self, name, type_var, impl_type, type_checker): self.type_checker = type_checker def __repr__(self): - return "%s[%s]" % (self.name, _type_repr(self.type_var)) + return f"{self.name}[{_type_repr(self.type_var)}]" def __getitem__(self, parameter): if not isinstance(self.type_var, TypeVar): - raise TypeError("%s cannot be further parameterized." % self) + raise TypeError(f"{self} cannot be further parameterized.") if self.type_var.__constraints__ and isinstance(parameter, type): if not issubclass(parameter, self.type_var.__constraints__): - raise TypeError("%s is not a valid substitution for %s." % - (parameter, self.type_var)) + raise TypeError(f"{parameter} is not a valid substitution for {self.type_var}.") if isinstance(parameter, TypeVar) and parameter is not self.type_var: - raise TypeError("%s cannot be re-parameterized." % self) + raise TypeError(f"{self} cannot be re-parameterized.") return self.__class__(self.name, parameter, self.impl_type, self.type_checker) From 4a0bf109456e4fb4faaef132ce40bc0fabfcea72 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 13 Nov 2017 09:47:18 +0100 Subject: [PATCH 34/82] Some more ideas --- Lib/typing.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/typing.py b/Lib/typing.py index 622819113f403f..0334f2d6bba1b0 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -214,7 +214,7 @@ def _remove_dups_flatten(parameters): # (In particular, Union[str, AnyStr] != AnyStr.) all_params = set(params) for t1 in params: - if not isinstance(t1, (type, _GenericAlias)): + if not isinstance(t1, type): continue if any(isinstance(t2, type) and issubclass(t1, t2) for t2 in all_params - {t1}): @@ -707,12 +707,12 @@ def __mro_entry__(self, bases): return self.__origin__ def __getattr__(self, attr): - if '__origin__' in self.__dict__: # We are carefull for copy and pickle + if '__origin__' in self.__dict__ and not(attr.startswith('__') and attr.endswith('__')): # We are carefull for copy and pickle return getattr(self.__origin__, attr) raise AttributeError(attr) def __setattr__(self, attr, val): - if attr not in ('__origin__', '__args__', '__parameters__'): + if not(attr.startswith('__') and attr.endswith('__')): setattr(self.__origin__, attr, val) self.__dict__[attr] = val From 152d2f6352d3eb70b3b364d4ee020b1bd126663e Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 13 Nov 2017 12:43:26 +0100 Subject: [PATCH 35/82] Allow expansion to multiple bases --- Lib/test/test_genericclass.py | 8 ++++---- Lib/test/test_types.py | 4 ++-- Lib/types.py | 7 +++++-- Python/bltinmodule.c | 20 ++++++++++++++------ 4 files changed, 25 insertions(+), 14 deletions(-) diff --git a/Lib/test/test_genericclass.py b/Lib/test/test_genericclass.py index fde788008cc664..8ad4d8f79f1834 100644 --- a/Lib/test/test_genericclass.py +++ b/Lib/test/test_genericclass.py @@ -42,7 +42,7 @@ class B: ... class C: def __mro_entry__(self, bases): tested.append(bases) - return None + return () c = C() self.assertEqual(tested, []) class D(A, c, B): ... @@ -88,13 +88,13 @@ class D(c, dict): ... def test_mro_entry_errors(self): class C_too_many: def __mro_entry__(self, bases, something, other): - return None + return () c = C_too_many() with self.assertRaises(TypeError): class D(c): ... class C_too_few: def __mro_entry__(self): - return None + return () d = C_too_few() with self.assertRaises(TypeError): class D(d): ... @@ -132,7 +132,7 @@ def test_mro_entry_type_call(self): # Substitution should _not_ happen in direct type call class C: def __mro_entry__(self, bases): - return None + return () c = C() with self.assertRaisesRegex(TypeError, "MRO entry resolution; " diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index f77a8bb363118e..b7dd83f50de44d 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -860,7 +860,7 @@ class A: pass class B: pass class C: def __mro_entry__(self, bases): - return None + return () c = C() D = types.new_class('D', (A, c, B), {}) self.assertEqual(D.__bases__, (A, B)) @@ -915,7 +915,7 @@ class B: pass class C: def __mro_entry__(self, bases): if A in bases: - return None + return () return A c = C() self.assertEqual(types.resolve_bases(()), ()) diff --git a/Lib/types.py b/Lib/types.py index c1ed6bdc8b8066..ef97da2df02cd1 100644 --- a/Lib/types.py +++ b/Lib/types.py @@ -79,10 +79,13 @@ def resolve_bases(bases): continue new_base = base.__mro_entry__(bases) updated = True - new_bases[i] = new_base + if not isinstance(new_base, tuple): + new_bases[i] = new_base + else: + new_bases[i:i+1] = new_base if not updated: return bases - return tuple(b for b in new_bases if b is not None) + return tuple(new_bases) def prepare_class(name, bases=(), kwds=None): """Call the __prepare__ method of the appropriate metaclass. diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 26c112b0d999e5..ce209c540a9987 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -49,8 +49,8 @@ _Py_IDENTIFIER(stderr); static PyObject* update_bases(PyObject* bases, PyObject** args, int nargs, int* modified_bases) { - int i, ind, tot_nones = 0; - PyObject *base, *meth, *new_base, *new_bases; + int i, j, ind, tot_extra = 0; + PyObject *base, *meth, *new_base, *new_sub_base, *new_bases; PyObject *stack[1] = {bases}; assert(PyTuple_Check(bases)); @@ -78,8 +78,8 @@ update_bases(PyObject* bases, PyObject** args, int nargs, int* modified_bases) Py_DECREF(meth); return NULL; } - if (new_base == Py_None) { - tot_nones++; + if (PyTuple_Check(new_base)) { + tot_extra += PyTuple_Size(new_base) - 1; } Py_DECREF(base); args[i] = new_base; @@ -89,15 +89,23 @@ update_bases(PyObject* bases, PyObject** args, int nargs, int* modified_bases) if (!*modified_bases){ return bases; } - new_bases = PyTuple_New(nargs - 2 - tot_nones); + new_bases = PyTuple_New(nargs - 2 + tot_extra); ind = 0; for (i = 2; i < nargs; i++) { new_base = args[i]; - if (new_base != Py_None) { + if (!PyTuple_Check(new_base)) { Py_INCREF(new_base); PyTuple_SET_ITEM(new_bases, ind, new_base); ind++; } + else { + for (j = 0; j < PyTuple_Size(new_base); j++) { + new_sub_base = PyTuple_GET_ITEM(new_base, j); + Py_INCREF(new_sub_base); + PyTuple_SET_ITEM(new_bases, ind, new_sub_base); + ind++; + } + } } return new_bases; } From 6788cbaf524089939b9012576a2c799da04ce514 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 13 Nov 2017 14:16:22 +0100 Subject: [PATCH 36/82] Update __mro_emtry__ in typing --- Lib/typing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/typing.py b/Lib/typing.py index 0334f2d6bba1b0..5afd89c29e4d74 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -703,7 +703,7 @@ def __mro_entry__(self, bases): i = bases.index(self) for b in bases[i+1:]: if isinstance(b, _GenericAlias): - return None + return () return self.__origin__ def __getattr__(self, attr): From 57da0177f51b54b7d384bf4aa1fa6f02581869a1 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 13 Nov 2017 17:19:06 +0100 Subject: [PATCH 37/82] Start using new __mro_entry__ --- Lib/typing.py | 245 +++++++++----------------------------------------- 1 file changed, 40 insertions(+), 205 deletions(-) diff --git a/Lib/typing.py b/Lib/typing.py index 5afd89c29e4d74..723a693278b9fb 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -84,10 +84,6 @@ # namespace, but excluded from __all__ because they might stomp on # legitimate imports of those modules. -# -# Internal helper functions. -# - def _trim_name(nm): whitelist = ('_TypingBase', '_FinalTypingBase', '_SingletonTypingBase', '_ForwardRef') if nm.startswith('_') and nm not in whitelist: @@ -253,9 +249,6 @@ def inner(*args, **kwds): return func(*args, **kwds) return inner -# -# Internal marker base classes: _TypingBase, _FinalTypingBase, _SingletonTypingBase -# class _TypingBase: """Internal indicator of special typing constructs.""" @@ -323,10 +316,6 @@ def __new__(cls, *args, _root=False, **kwds): def __reduce__(self): return _trim_name(type(self).__name__) -# -# Final classes: ForwardRef and TypeVar. These should not be subclassed, -# but can be instantiated. -# class _ForwardRef(_FinalTypingBase, _root=True): """Internal wrapper to hold a forward reference.""" @@ -465,10 +454,6 @@ def __repr__(self): # (This one *is* for export!) AnyStr = TypeVar('AnyStr', bytes, str) -# -# Singleton classes: Any and NoReturn. -# These should not be neither subclassed, nor instantiated. -# class _Any(_SingletonTypingBase, _root=True): """Special type indicating an unconstrained type. @@ -507,12 +492,6 @@ def stop() -> NoReturn: NoReturn = _NoReturn(_root=True) -# -# Subscriptable singleton classes: ClassVar, Union, and Optional. -# Like above, but can be subscripted. -# - - class _ClassVar(_SingletonTypingBase, _root=True): """Special type construct to mark class variables. @@ -1227,32 +1206,13 @@ def __class_getitem__(cls, params): # Various ABCs mimicking those in collections.abc. -# A few are simply re-exported for completeness. - -Hashable = collections.abc.Hashable # Not generic. - - -class Awaitable(collections.abc.Awaitable, Generic[T_co]): - __slots__ = () - - -class Coroutine(collections.abc.Coroutine, Generic[T_co, T_contra, V_co]): - __slots__ = () - -class AsyncIterable(collections.abc.AsyncIterable, Generic[T_co]): - __slots__ = () - - -class AsyncIterator(collections.abc.AsyncIterator, Generic[T_co]): - __slots__ = () - - -class Iterable(collections.abc.Iterable, Generic[T_co]): - __slots__ = () - - -class Iterator(collections.abc.Iterator, Generic[T_co]): - __slots__ = () +Hashable = _GenericAlias(collections.abc.Hashable, []) # Not generic. +Awaitable = _GenericAlias(collections.abc.Awaitable, [T_co]) +Coroutine = _GenericAlias(collections.abc.Coroutine, [T_co, T_contra, V_co]) +AsyncIterable = _GenericAlias(collections.abc.AsyncIterable, [T_co]) +AsyncIterator = _GenericAlias(collections.abc.AsyncIterator, [T_co]) +Iterable = _GenericAlias(collections.abc.Iterable, [T_co]) +Iterator = _GenericAlias(collections.abc.Iterator, [T_co]) class SupportsInt(_Protocol): @@ -1303,166 +1263,41 @@ def __round__(self, ndigits: int = 0) -> T_co: pass -class Reversible(collections.abc.Reversible, Generic[T_co]): - __slots__ = () - - -Sized = collections.abc.Sized # Not generic. - - -class Container(collections.abc.Container, Generic[T_co]): - __slots__ = () - - -class Collection(collections.abc.Collection, Generic[T_co]): - __slots__ = () - - +Reversible = _GenericAlias(collections.abc.Reversible, [T_co]) +Sized = _GenericAlias(collections.abc.Sized, []) # Not generic. +Container = _GenericAlias(collections.abc.Container, [T_co]) +Collection = _GenericAlias(collections.abc.Collection, [T_co]) # Callable was defined earlier. - -class AbstractSet(collections.abc.Set, Generic[T_co]): - __slots__ = () - - -class MutableSet(collections.abc.MutableSet, Generic[T]): - __slots__ = () - - -# NOTE: It is only covariant in the value type. -class Mapping(collections.abc.Mapping, Generic[KT, VT_co]): - __slots__ = () - - -class MutableMapping(collections.abc.MutableMapping, Generic[KT, VT]): - __slots__ = () - - -class Sequence(collections.abc.Sequence, Generic[T_co]): - __slots__ = () - - -class MutableSequence(collections.abc.MutableSequence, Generic[T]): - __slots__ = () - - +AbstractSet = _GenericAlias(collections.abc.Set, [T_co]) +MutableSet = _GenericAlias(collections.abc.MutableSet, [T]) +# NOTE: Mapping is only covariant in the value type. +Mapping = _GenericAlias(collections.abc.Mapping, [KT, VT_co]) +MutableMapping = _GenericAlias(collections.abc.MutableMapping, [KT, VT]) +Sequence = _GenericAlias(collections.abc.Sequence, [T_co]) +MutableSequence = _GenericAlias(collections.abc.MutableSequence, [T]) # Not generic -ByteString = collections.abc.ByteString - - -class List(list, Generic[T], metaclass=abc.ABCMeta): - __slots__ = () - - def __new__(cls, *args, **kwds): - if cls is List: - raise TypeError("Type List cannot be instantiated; " - "use list() instead") - return super().__new__(cls, *args, **kwds) - - -class Deque(collections.deque, Generic[T], metaclass=abc.ABCMeta): - __slots__ = () - - def __new__(cls, *args, **kwargs): - if cls is Deque: - return collections.deque(*args, **kwargs) - return super().__new__(cls, *args, **kwargs) - - - -class Set(set, Generic[T], metaclass=abc.ABCMeta): - __slots__ = () - - def __new__(cls, *args, **kwds): - if cls is Set: - raise TypeError("Type Set cannot be instantiated; " - "use set() instead") - return super().__new__(cls, *args, **kwds) - - -class FrozenSet(frozenset, Generic[T_co], metaclass=abc.ABCMeta): - __slots__ = () - - def __new__(cls, *args, **kwds): - if cls is FrozenSet: - raise TypeError("Type FrozenSet cannot be instantiated; " - "use frozenset() instead") - return super().__new__(cls, *args, **kwds) - - -class MappingView(collections.abc.MappingView, Generic[T_co]): - __slots__ = () - - -class KeysView(collections.abc.KeysView, Generic[KT]): - __slots__ = () - - -class ItemsView(collections.abc.ItemsView, Generic[KT, VT_co]): - __slots__ = () - - -class ValuesView(collections.abc.ValuesView, Generic[VT_co]): - __slots__ = () - - -class ContextManager(contextlib.AbstractContextManager, Generic[T_co]): - __slots__ = () - - -#class AsyncContextManager(contextlib.AbstractAsyncContextManager, Generic[T_co]): -# __slots__ = () - - -class Dict(dict, Generic[KT, VT], metaclass=abc.ABCMeta): - __slots__ = () - - def __new__(cls, *args, **kwds): - if cls is Dict: - raise TypeError("Type Dict cannot be instantiated; " - "use dict() instead") - return super().__new__(cls, *args, **kwds) - - -class DefaultDict(collections.defaultdict, Generic[KT, VT], metaclass=abc.ABCMeta): - __slots__ = () - - def __new__(cls, *args, **kwargs): - if cls is DefaultDict: - return collections.defaultdict(*args, **kwargs) - return super().__new__(cls, *args, **kwargs) - - -class Counter(collections.Counter, Generic[T], metaclass=abc.ABCMeta): - __slots__ = () - - def __new__(cls, *args, **kwargs): - if cls is Counter: - return collections.Counter(*args, **kwargs) - return super().__new__(cls, *args, **kwargs) - - -class ChainMap(collections.ChainMap, Generic[KT, VT], metaclass=abc.ABCMeta): - __slots__ = () - - def __new__(cls, *args, **kwargs): - if cls is ChainMap: - return collections.ChainMap(*args, **kwargs) - return super().__new__(cls, *args, **kwargs) - - -class Generator(collections.abc.Generator, Generic[T_co, T_contra, V_co]): - __slots__ = () - - def __new__(cls, *args, **kwds): - if cls is Generator: - raise TypeError("Type Generator cannot be instantiated; " - "create a subclass instead") - return super().__new__(cls, *args, **kwds) - - -class AsyncGenerator(collections.abc.AsyncGenerator, Generic[T_co, T_contra]): - __slots__ = () - +ByteString = _GenericAlias(collections.abc.ByteString, []) +List = _GenericAlias(list, Generic[T]) + +if False: + raise TypeError("Type List cannot be instantiated; " + "use list() instead") + +Deque = _GenericAlias(collections.deque, [T]) +Set = _GenericAlias(set, [T]) +FrozenSet = _GenericAlias(frozenset, [T_co]) +MappingView = _GenericAlias(collections.abc.MappingView, [T_co]) +KeysView = _GenericAlias(collections.abc.KeysView, [KT]) +ItemsView = _GenericAlias(collections.abc.ItemsView, [KT, VT_co]) +ValuesView = _GenericAlias(collections.abc.ValuesView, [VT_co]) +ContextManager = _GenericAlias(contextlib.AbstractContextManager, [T_co]) +#AsyncContextManager = _GenericAlias(contextlib.AbstractAsyncContextManager, [T_co]) +Dict = _GenericAlias(dict, [KT, VT]) +DefaultDict = _GenericAlias(collections.defaultdict, [KT, VT]) +Counter = _GenericAlias(collections.Counter, [T]) +ChainMap = _GenericAlias(collections.ChainMap, [KT, VT]) +Generator = _GenericAlias(collections.abc.Generator, [T_co, T_contra, V_co]) +AsyncGenerator = _GenericAlias(collections.abc.AsyncGenerator, [T_co, T_contra]) # Internal type variable used for Type[]. CT_co = TypeVar('CT_co', covariant=True, bound=type) From cf016fbad73e515925db600ecd1884dc68015faa Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 14 Nov 2017 00:36:38 +0100 Subject: [PATCH 38/82] Some progress towards new scheme --- Lib/typing.py | 131 +++++++++++++++++++++++++++++++------------------- 1 file changed, 81 insertions(+), 50 deletions(-) diff --git a/Lib/typing.py b/Lib/typing.py index 723a693278b9fb..6280c6014867dc 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -176,7 +176,10 @@ def _subs_tvars(tp, tvars, subs): new_args[a] = _subs_tvars(arg, tvars, subs) if tp.__origin__ is Union: return Union[tuple(new_args)] - return _GenericAlias(tp.__origin__, tuple(new_args)) + result = _GenericAlias(tp.__origin__, tuple(new_args), name=tp._name) + if hasattr(tp, '_inst'): + result._inst = tp._inst + return result def _remove_dups_flatten(parameters): @@ -612,18 +615,25 @@ def __getitem__(self, arg): class _GenericAlias(_FinalTypingBase, _root=True): - def __init__(self, origin, params): + def __init__(self, origin, params, *, name=None, subcls=True, inst=True): """Create a new generic class. GenericMeta.__new__ accepts keyword arguments that are used for internal bookkeeping, therefore an override should pass unused keyword arguments to super(). """ - if not isinstance(params, tuple): + self._name = name + if not isinstance(params, (tuple, list)): params = (params,) self.__origin__ = origin - self.__args__ = tuple(... if a is _TypingEllipsis else - () if a is _TypingEmpty else - a for a in params) - self.__parameters__ = _type_vars(params) + if name: + self.__args__ = () + self.__parameters__ = tuple(params) + self._subcls = subcls + self._inst = inst + else: + self.__args__ = tuple(... if a is _TypingEllipsis else + () if a is _TypingEmpty else + a for a in params) + self.__parameters__ = _type_vars(params) def _get_type_vars(self, tvars): _get_type_vars(self.__parameters__, tvars) @@ -644,13 +654,25 @@ def __getitem__(self, params): msg = "Parameters to generic types must be types." params = tuple(_type_check(p, msg) for p in params) _check_generic(self, params) - return _subs_tvars(self, self.__parameters__, params) + if self._name and not self.__args__: + origin = _GenericAlias(self.__origin__, self.__parameters__, name=self._name) + origin.__args__ = tuple(origin.__parameters__) + else: + origin = self + return _subs_tvars(origin, self.__parameters__, params) def __repr__(self): if (self.__origin__ is not Callable or len(self.__args__) == 2 and self.__args__[0] is Ellipsis): - return (f'{_type_repr(self.__origin__)}' - f'[{", ".join([_type_repr(a) for a in self.__args__])}]') + if self._name: + name = 'typing.' + self._name + else: + name = _type_repr(self.__origin__) + if self.__args__: + args = f'[{", ".join([_type_repr(a) for a in self.__args__])}]' + else: + args = '' + return (f'{name}{args}') return (f'typing.Callable' f'[[{", ".join([_type_repr(a) for a in self.__args__[:-1]])}], ' f'{_type_repr(self.__args__[-1])}]') @@ -670,6 +692,9 @@ def __hash__(self): return hash((self.__origin__, self.__args__)) def __call__(self, *args, **kwargs): + if self._name and not self._inst: + raise TypeError(f"Type {self._name} cannot be instantiated; " + f"use {self._name.lower()}() instead") result = self.__origin__(*args, **kwargs) try: result.__orig_class__ = self @@ -678,6 +703,8 @@ def __call__(self, *args, **kwargs): return result def __mro_entry__(self, bases): + if self._name: + return (self.__origin__, Generic) if self.__origin__ is Generic: i = bases.index(self) for b in bases[i+1:]: @@ -691,10 +718,19 @@ def __getattr__(self, attr): raise AttributeError(attr) def __setattr__(self, attr, val): - if not(attr.startswith('__') and attr.endswith('__')): + if not(attr.startswith('__') and attr.endswith('__') or attr in ('_name', '_subcls', '_inst')): setattr(self.__origin__, attr, val) self.__dict__[attr] = val + def __instancecheck__(self, obj): + return self.__subclasscheck__(type(obj)) + + def __subclasscheck__(self, cls): + if not self.__args__: + return issubclass(cls, self.__origin__) + else: + raise TypeError("Subscripted generic cannot be used with class and instance checks") + def _make_subclasshook(cls): """Construct a __subclasshook__ callable that incorporates @@ -1206,13 +1242,13 @@ def __class_getitem__(cls, params): # Various ABCs mimicking those in collections.abc. -Hashable = _GenericAlias(collections.abc.Hashable, []) # Not generic. -Awaitable = _GenericAlias(collections.abc.Awaitable, [T_co]) -Coroutine = _GenericAlias(collections.abc.Coroutine, [T_co, T_contra, V_co]) -AsyncIterable = _GenericAlias(collections.abc.AsyncIterable, [T_co]) -AsyncIterator = _GenericAlias(collections.abc.AsyncIterator, [T_co]) -Iterable = _GenericAlias(collections.abc.Iterable, [T_co]) -Iterator = _GenericAlias(collections.abc.Iterator, [T_co]) +Hashable = _GenericAlias(collections.abc.Hashable, [], name='Hashable') # Not generic. +Awaitable = _GenericAlias(collections.abc.Awaitable, [T_co], name='Awaitable') +Coroutine = _GenericAlias(collections.abc.Coroutine, [T_co, T_contra, V_co], name='Coroutine') +AsyncIterable = _GenericAlias(collections.abc.AsyncIterable, [T_co], name='AsyncIterable') +AsyncIterator = _GenericAlias(collections.abc.AsyncIterator, [T_co], name='AsyncIterator') +Iterable = _GenericAlias(collections.abc.Iterable, [T_co], name='Iterable') +Iterator = _GenericAlias(collections.abc.Iterator, [T_co], name='Iterator') class SupportsInt(_Protocol): @@ -1263,41 +1299,36 @@ def __round__(self, ndigits: int = 0) -> T_co: pass -Reversible = _GenericAlias(collections.abc.Reversible, [T_co]) -Sized = _GenericAlias(collections.abc.Sized, []) # Not generic. -Container = _GenericAlias(collections.abc.Container, [T_co]) -Collection = _GenericAlias(collections.abc.Collection, [T_co]) +Reversible = _GenericAlias(collections.abc.Reversible, [T_co], name='Reversible') +Sized = _GenericAlias(collections.abc.Sized, [], name='Sized') # Not generic. +Container = _GenericAlias(collections.abc.Container, [T_co], name='Container') +Collection = _GenericAlias(collections.abc.Collection, [T_co], name='Collection') # Callable was defined earlier. -AbstractSet = _GenericAlias(collections.abc.Set, [T_co]) -MutableSet = _GenericAlias(collections.abc.MutableSet, [T]) +AbstractSet = _GenericAlias(collections.abc.Set, [T_co], name='AbstractSet') +MutableSet = _GenericAlias(collections.abc.MutableSet, [T], name='MutableSet') # NOTE: Mapping is only covariant in the value type. -Mapping = _GenericAlias(collections.abc.Mapping, [KT, VT_co]) -MutableMapping = _GenericAlias(collections.abc.MutableMapping, [KT, VT]) -Sequence = _GenericAlias(collections.abc.Sequence, [T_co]) -MutableSequence = _GenericAlias(collections.abc.MutableSequence, [T]) +Mapping = _GenericAlias(collections.abc.Mapping, [KT, VT_co], name='Mapping') +MutableMapping = _GenericAlias(collections.abc.MutableMapping, [KT, VT], name='MutableMapping') +Sequence = _GenericAlias(collections.abc.Sequence, [T_co], name='Sequence') +MutableSequence = _GenericAlias(collections.abc.MutableSequence, [T], name='MutableSequence') # Not generic -ByteString = _GenericAlias(collections.abc.ByteString, []) -List = _GenericAlias(list, Generic[T]) - -if False: - raise TypeError("Type List cannot be instantiated; " - "use list() instead") - -Deque = _GenericAlias(collections.deque, [T]) -Set = _GenericAlias(set, [T]) -FrozenSet = _GenericAlias(frozenset, [T_co]) -MappingView = _GenericAlias(collections.abc.MappingView, [T_co]) -KeysView = _GenericAlias(collections.abc.KeysView, [KT]) -ItemsView = _GenericAlias(collections.abc.ItemsView, [KT, VT_co]) -ValuesView = _GenericAlias(collections.abc.ValuesView, [VT_co]) -ContextManager = _GenericAlias(contextlib.AbstractContextManager, [T_co]) -#AsyncContextManager = _GenericAlias(contextlib.AbstractAsyncContextManager, [T_co]) -Dict = _GenericAlias(dict, [KT, VT]) -DefaultDict = _GenericAlias(collections.defaultdict, [KT, VT]) -Counter = _GenericAlias(collections.Counter, [T]) -ChainMap = _GenericAlias(collections.ChainMap, [KT, VT]) -Generator = _GenericAlias(collections.abc.Generator, [T_co, T_contra, V_co]) -AsyncGenerator = _GenericAlias(collections.abc.AsyncGenerator, [T_co, T_contra]) +ByteString = _GenericAlias(collections.abc.ByteString, [], name='ByteString') +List = _GenericAlias(list, [T], name='List', inst=False) +Deque = _GenericAlias(collections.deque, [T], name='Deque') +Set = _GenericAlias(set, [T], name='Set', inst=False) +FrozenSet = _GenericAlias(frozenset, [T_co], name='FrozenSet', inst=False) +MappingView = _GenericAlias(collections.abc.MappingView, [T_co], name='MappingView') +KeysView = _GenericAlias(collections.abc.KeysView, [KT], name='KeysView') +ItemsView = _GenericAlias(collections.abc.ItemsView, [KT, VT_co], name='ItemsView') +ValuesView = _GenericAlias(collections.abc.ValuesView, [VT_co], name='ValuesView') +ContextManager = _GenericAlias(contextlib.AbstractContextManager, [T_co], name='ContextManager') +#AsyncContextManager = _GenericAlias(contextlib.AbstractAsyncContextManager, [T_co], name='AsyncContextManager') +Dict = _GenericAlias(dict, [KT, VT], name='Dict', inst=False) +DefaultDict = _GenericAlias(collections.defaultdict, [KT, VT], name='DefaultDict') +Counter = _GenericAlias(collections.Counter, [T], name='Counter') +ChainMap = _GenericAlias(collections.ChainMap, [KT, VT], name='ChainMap') +Generator = _GenericAlias(collections.abc.Generator, [T_co, T_contra, V_co], name='Generator') +AsyncGenerator = _GenericAlias(collections.abc.AsyncGenerator, [T_co, T_contra], name='AsyncGenerator') # Internal type variable used for Type[]. CT_co = TypeVar('CT_co', covariant=True, bound=type) From f581aa30b4b0269e9cb1708c57f37ed3347e9f2d Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 14 Nov 2017 10:21:42 +0100 Subject: [PATCH 39/82] Some fixes; use _special --- Lib/test/test_typing.py | 19 +++--- Lib/typing.py | 133 +++++++++++++++++++--------------------- 2 files changed, 75 insertions(+), 77 deletions(-) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 1fb01bbc3d11a4..c40e6d68f136c4 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -594,7 +594,9 @@ def test_basics(self): Y[str] with self.assertRaises(TypeError): Y[str, str] - self.assertIsSubclass(SimpleMapping[str, int], SimpleMapping) + with self.assertRaises(TypeError): + issubclass(SimpleMapping[str, int], SimpleMapping) + self.assertIsInstance(SimpleMapping[str, int](), SimpleMapping) def test_generic_errors(self): T = TypeVar('T') @@ -659,7 +661,7 @@ def test_new_repr(self): U = TypeVar('U', covariant=True) S = TypeVar('S') - self.assertEqual(repr(List), "") + self.assertEqual(repr(List), 'typing.List') self.assertEqual(repr(List[T]), 'typing.List[~T]') self.assertEqual(repr(List[U]), 'typing.List[+U]') self.assertEqual(repr(List[S][T][int]), 'typing.List[int]') @@ -822,8 +824,8 @@ class D(C, List[T][U][V]): ... self.assertEqual(D[int].__parameters__, ()) self.assertEqual(C[int].__args__, (int,)) self.assertEqual(D[int].__args__, (int,)) - self.assertEqual(C.__bases__, (List,)) - self.assertEqual(D.__bases__, (C, List)) + self.assertEqual(C.__bases__, (list, Generic)) + self.assertEqual(D.__bases__, (C, list, Generic)) self.assertEqual(C.__orig_bases__, (List[T][U][V],)) self.assertEqual(D.__orig_bases__, (C, List[T][U][V])) @@ -953,8 +955,10 @@ def __call__(self): self.assertIsSubclass(C1, collections.abc.Callable) self.assertIsInstance(T1(), tuple) self.assertIsSubclass(T2, tuple) - self.assertIsSubclass(Tuple[int, ...], typing.Sequence) - self.assertIsSubclass(Tuple[int, ...], typing.Iterable) + with self.assertRaises(TypeError): + issubclass(Tuple[int, ...], typing.Sequence) + with self.assertRaises(TypeError): + issubclass(Tuple[int, ...], typing.Iterable) def test_fail_with_bare_union(self): with self.assertRaises(TypeError): @@ -2078,7 +2082,8 @@ def __len__(self): self.assertIsSubclass(MMB, collections.abc.Mapping) self.assertIsSubclass(MMC, collections.abc.Mapping) - self.assertIsSubclass(MMB[str, str], typing.Mapping) + with self.assertRaises(TypeError): + issubclass(MMB[str, str], typing.Mapping) self.assertIsSubclass(MMC, MMA) class I(typing.Iterable): ... diff --git a/Lib/typing.py b/Lib/typing.py index 6280c6014867dc..ac0868e393e5f8 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -176,10 +176,8 @@ def _subs_tvars(tp, tvars, subs): new_args[a] = _subs_tvars(arg, tvars, subs) if tp.__origin__ is Union: return Union[tuple(new_args)] - result = _GenericAlias(tp.__origin__, tuple(new_args), name=tp._name) - if hasattr(tp, '_inst'): - result._inst = tp._inst - return result + return _GenericAlias(tp.__origin__, tuple(new_args), + name=tp._name, subcls=tp._subcls, inst=tp._inst) def _remove_dups_flatten(parameters): @@ -215,7 +213,7 @@ def _remove_dups_flatten(parameters): for t1 in params: if not isinstance(t1, type): continue - if any(isinstance(t2, type) and issubclass(t1, t2) + if any((isinstance(t2, type) or isinstance(t2, _GenericAlias) and t2._special) and issubclass(t1, t2) for t2 in all_params - {t1}): all_params.remove(t1) return tuple(t for t in params if t in all_params) @@ -612,37 +610,37 @@ def __getitem__(self, arg): # * __args__ is a tuple of all arguments used in subscripting, # e.g., Dict[T, int].__args__ == (T, int). +_FLAGS = ('_name', '_subcls', '_inst', '_special') + +def _is_dunder(attr): + return attr.startswith('__') and attr.endswith('__') + class _GenericAlias(_FinalTypingBase, _root=True): - def __init__(self, origin, params, *, name=None, subcls=True, inst=True): - """Create a new generic class. GenericMeta.__new__ accepts - keyword arguments that are used for internal bookkeeping, therefore - an override should pass unused keyword arguments to super(). - """ + def __init__(self, origin, params, *, name=None, subcls=True, inst=True, special=False): self._name = name - if not isinstance(params, (tuple, list)): + self._subcls = subcls + self._inst = inst + self._special = special + if not isinstance(params, tuple): params = (params,) self.__origin__ = origin - if name: - self.__args__ = () - self.__parameters__ = tuple(params) - self._subcls = subcls - self._inst = inst - else: - self.__args__ = tuple(... if a is _TypingEllipsis else - () if a is _TypingEmpty else - a for a in params) - self.__parameters__ = _type_vars(params) + self.__args__ = tuple(... if a is _TypingEllipsis else + () if a is _TypingEmpty else + a for a in params) + self.__parameters__ = _type_vars(params) def _get_type_vars(self, tvars): - _get_type_vars(self.__parameters__, tvars) + if not self._special: + _get_type_vars(self.__parameters__, tvars) def _eval_type(self, globalns, localns): ev_args = tuple(_eval_type(a, globalns, localns) for a in self.__args__) if ev_args == self.__args__: return self - return _GenericAlias(self.__origin__, ev_args) + return _GenericAlias(self.__origin__, ev_args, name=self._name, + subcls=self._subcls, inst=self._inst, special=self._special) @_tp_cache def __getitem__(self, params): @@ -654,12 +652,7 @@ def __getitem__(self, params): msg = "Parameters to generic types must be types." params = tuple(_type_check(p, msg) for p in params) _check_generic(self, params) - if self._name and not self.__args__: - origin = _GenericAlias(self.__origin__, self.__parameters__, name=self._name) - origin.__args__ = tuple(origin.__parameters__) - else: - origin = self - return _subs_tvars(origin, self.__parameters__, params) + return _subs_tvars(self, self.__parameters__, params) def __repr__(self): if (self.__origin__ is not Callable or @@ -668,7 +661,7 @@ def __repr__(self): name = 'typing.' + self._name else: name = _type_repr(self.__origin__) - if self.__args__: + if not self._special: args = f'[{", ".join([_type_repr(a) for a in self.__args__])}]' else: args = '' @@ -692,7 +685,7 @@ def __hash__(self): return hash((self.__origin__, self.__args__)) def __call__(self, *args, **kwargs): - if self._name and not self._inst: + if not self._inst: raise TypeError(f"Type {self._name} cannot be instantiated; " f"use {self._name.lower()}() instead") result = self.__origin__(*args, **kwargs) @@ -713,12 +706,12 @@ def __mro_entry__(self, bases): return self.__origin__ def __getattr__(self, attr): - if '__origin__' in self.__dict__ and not(attr.startswith('__') and attr.endswith('__')): # We are carefull for copy and pickle + if '__origin__' in self.__dict__ and not _is_dunder(attr): # We are carefull for copy and pickle return getattr(self.__origin__, attr) raise AttributeError(attr) def __setattr__(self, attr, val): - if not(attr.startswith('__') and attr.endswith('__') or attr in ('_name', '_subcls', '_inst')): + if not _is_dunder(attr) and attr not in _FLAGS: setattr(self.__origin__, attr, val) self.__dict__[attr] = val @@ -726,7 +719,7 @@ def __instancecheck__(self, obj): return self.__subclasscheck__(type(obj)) def __subclasscheck__(self, cls): - if not self.__args__: + if self._special: return issubclass(cls, self.__origin__) else: raise TypeError("Subscripted generic cannot be used with class and instance checks") @@ -823,11 +816,11 @@ def __class_getitem__(cls, params): def __init_subclass__(cls, *args, **kwargs): tvars = [] - if (not hasattr(cls, '__orig_bases__') and Generic in cls.__bases__ and + if (not '__orig_bases__' in cls.__dict__ and Generic in cls.__bases__ and cls.__name__ not in ('Tuple', 'Callable') or hasattr(cls, '__orig_bases__') and Generic in cls.__orig_bases__): raise TypeError("Cannot inherit from plain Generic") - if hasattr(cls, '__orig_bases__'): + if '__orig_bases__' in cls.__dict__: tvars = _type_vars(cls.__orig_bases__) # Look for Generic[T1, ..., Tn]. # If found, tvars must be a subset of it. @@ -1242,13 +1235,13 @@ def __class_getitem__(cls, params): # Various ABCs mimicking those in collections.abc. -Hashable = _GenericAlias(collections.abc.Hashable, [], name='Hashable') # Not generic. -Awaitable = _GenericAlias(collections.abc.Awaitable, [T_co], name='Awaitable') -Coroutine = _GenericAlias(collections.abc.Coroutine, [T_co, T_contra, V_co], name='Coroutine') -AsyncIterable = _GenericAlias(collections.abc.AsyncIterable, [T_co], name='AsyncIterable') -AsyncIterator = _GenericAlias(collections.abc.AsyncIterator, [T_co], name='AsyncIterator') -Iterable = _GenericAlias(collections.abc.Iterable, [T_co], name='Iterable') -Iterator = _GenericAlias(collections.abc.Iterator, [T_co], name='Iterator') +Hashable = _GenericAlias(collections.abc.Hashable, (), name='Hashable', special=True) # Not generic. +Awaitable = _GenericAlias(collections.abc.Awaitable, T_co, name='Awaitable', special=True) +Coroutine = _GenericAlias(collections.abc.Coroutine, (T_co, T_contra, V_co), name='Coroutine', special=True) +AsyncIterable = _GenericAlias(collections.abc.AsyncIterable, T_co, name='AsyncIterable', special=True) +AsyncIterator = _GenericAlias(collections.abc.AsyncIterator, T_co, name='AsyncIterator', special=True) +Iterable = _GenericAlias(collections.abc.Iterable, T_co, name='Iterable', special=True) +Iterator = _GenericAlias(collections.abc.Iterator, T_co, name='Iterator', special=True) class SupportsInt(_Protocol): @@ -1299,36 +1292,36 @@ def __round__(self, ndigits: int = 0) -> T_co: pass -Reversible = _GenericAlias(collections.abc.Reversible, [T_co], name='Reversible') -Sized = _GenericAlias(collections.abc.Sized, [], name='Sized') # Not generic. -Container = _GenericAlias(collections.abc.Container, [T_co], name='Container') -Collection = _GenericAlias(collections.abc.Collection, [T_co], name='Collection') +Reversible = _GenericAlias(collections.abc.Reversible, T_co, name='Reversible', special=True) +Sized = _GenericAlias(collections.abc.Sized, (), name='Sized', special=True) # Not generic. +Container = _GenericAlias(collections.abc.Container, T_co, name='Container', special=True) +Collection = _GenericAlias(collections.abc.Collection, T_co, name='Collection', special=True) # Callable was defined earlier. -AbstractSet = _GenericAlias(collections.abc.Set, [T_co], name='AbstractSet') -MutableSet = _GenericAlias(collections.abc.MutableSet, [T], name='MutableSet') +AbstractSet = _GenericAlias(collections.abc.Set, T_co, name='AbstractSet', special=True) +MutableSet = _GenericAlias(collections.abc.MutableSet, T, name='MutableSet', special=True) # NOTE: Mapping is only covariant in the value type. -Mapping = _GenericAlias(collections.abc.Mapping, [KT, VT_co], name='Mapping') -MutableMapping = _GenericAlias(collections.abc.MutableMapping, [KT, VT], name='MutableMapping') -Sequence = _GenericAlias(collections.abc.Sequence, [T_co], name='Sequence') -MutableSequence = _GenericAlias(collections.abc.MutableSequence, [T], name='MutableSequence') +Mapping = _GenericAlias(collections.abc.Mapping, (KT, VT_co), name='Mapping', special=True) +MutableMapping = _GenericAlias(collections.abc.MutableMapping, (KT, VT), name='MutableMapping', special=True) +Sequence = _GenericAlias(collections.abc.Sequence, T_co, name='Sequence', special=True) +MutableSequence = _GenericAlias(collections.abc.MutableSequence, T, name='MutableSequence', special=True) # Not generic -ByteString = _GenericAlias(collections.abc.ByteString, [], name='ByteString') -List = _GenericAlias(list, [T], name='List', inst=False) -Deque = _GenericAlias(collections.deque, [T], name='Deque') -Set = _GenericAlias(set, [T], name='Set', inst=False) -FrozenSet = _GenericAlias(frozenset, [T_co], name='FrozenSet', inst=False) -MappingView = _GenericAlias(collections.abc.MappingView, [T_co], name='MappingView') -KeysView = _GenericAlias(collections.abc.KeysView, [KT], name='KeysView') -ItemsView = _GenericAlias(collections.abc.ItemsView, [KT, VT_co], name='ItemsView') -ValuesView = _GenericAlias(collections.abc.ValuesView, [VT_co], name='ValuesView') -ContextManager = _GenericAlias(contextlib.AbstractContextManager, [T_co], name='ContextManager') -#AsyncContextManager = _GenericAlias(contextlib.AbstractAsyncContextManager, [T_co], name='AsyncContextManager') -Dict = _GenericAlias(dict, [KT, VT], name='Dict', inst=False) -DefaultDict = _GenericAlias(collections.defaultdict, [KT, VT], name='DefaultDict') -Counter = _GenericAlias(collections.Counter, [T], name='Counter') -ChainMap = _GenericAlias(collections.ChainMap, [KT, VT], name='ChainMap') -Generator = _GenericAlias(collections.abc.Generator, [T_co, T_contra, V_co], name='Generator') -AsyncGenerator = _GenericAlias(collections.abc.AsyncGenerator, [T_co, T_contra], name='AsyncGenerator') +ByteString = _GenericAlias(collections.abc.ByteString, (), name='ByteString', special=True) +List = _GenericAlias(list, T, name='List', inst=False, special=True) +Deque = _GenericAlias(collections.deque, T, name='Deque', special=True) +Set = _GenericAlias(set, T, name='Set', inst=False, special=True) +FrozenSet = _GenericAlias(frozenset, T_co, name='FrozenSet', inst=False, special=True) +MappingView = _GenericAlias(collections.abc.MappingView, T_co, name='MappingView', special=True) +KeysView = _GenericAlias(collections.abc.KeysView, KT, name='KeysView', special=True) +ItemsView = _GenericAlias(collections.abc.ItemsView, (KT, VT_co), name='ItemsView', special=True) +ValuesView = _GenericAlias(collections.abc.ValuesView, VT_co, name='ValuesView', special=True) +ContextManager = _GenericAlias(contextlib.AbstractContextManager, T_co, name='ContextManager', special=True) +#AsyncContextManager = _GenericAlias(contextlib.AbstractAsyncContextManager, T_co, name='AsyncContextManager') +Dict = _GenericAlias(dict, (KT, VT), name='Dict', inst=False, special=True) +DefaultDict = _GenericAlias(collections.defaultdict, (KT, VT), name='DefaultDict', special=True) +Counter = _GenericAlias(collections.Counter, T, name='Counter', special=True) +ChainMap = _GenericAlias(collections.ChainMap, (KT, VT), name='ChainMap', special=True) +Generator = _GenericAlias(collections.abc.Generator, (T_co, T_contra, V_co), name='Generator', special=True) +AsyncGenerator = _GenericAlias(collections.abc.AsyncGenerator, (T_co, T_contra), name='AsyncGenerator', special=True) # Internal type variable used for Type[]. CT_co = TypeVar('CT_co', covariant=True, bound=type) From 1a218e0c42faa7e8d9b34795dc6e3f2f4ccd5fb0 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 14 Nov 2017 10:31:07 +0100 Subject: [PATCH 40/82] Prohibit returning non-tuple --- Lib/test/test_genericclass.py | 16 +++++++++++----- Python/bltinmodule.c | 6 ++++++ 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_genericclass.py b/Lib/test/test_genericclass.py index 8ad4d8f79f1834..e810887b744094 100644 --- a/Lib/test/test_genericclass.py +++ b/Lib/test/test_genericclass.py @@ -8,7 +8,7 @@ class B: ... class C: def __mro_entry__(self, *args, **kwargs): tested.extend([args, kwargs]) - return C + return (C,) c = C() self.assertEqual(tested, []) class D(B, c): ... @@ -22,7 +22,7 @@ class B: ... class C: def __mro_entry__(self, bases): tested.append(bases) - return self.__class__ + return (self.__class__,) c = C() self.assertEqual(tested, []) class D(A, c, B): ... @@ -62,7 +62,7 @@ class A: ... class C: def __mro_entry__(self, bases): tested.append(bases) - return dict + return (dict,) c = C() self.assertEqual(tested, []) class D(A, c): ... @@ -76,7 +76,7 @@ def test_mro_entry_with_builtins_2(self): class C: def __mro_entry__(self, bases): tested.append(bases) - return C + return (C,) c = C() self.assertEqual(tested, []) class D(c, dict): ... @@ -105,6 +105,12 @@ class C_not_callable: c = C_not_callable() with self.assertRaises(TypeError): class D(c): ... + class C_not_tuple: + def __mro_entry__(self): + return object + c = C_not_tuple() + with self.assertRaises(TypeError): + class D(c): ... def test_mro_entry_metaclass(self): meta_args = [] @@ -115,7 +121,7 @@ def __new__(mcls, name, bases, ns): class A: ... class C: def __mro_entry__(self, bases): - return A + return (A,) c = C() class D(c, metaclass=Meta): x = 1 diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index ce209c540a9987..6b3f28f7fe1256 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -81,6 +81,12 @@ update_bases(PyObject* bases, PyObject** args, int nargs, int* modified_bases) if (PyTuple_Check(new_base)) { tot_extra += PyTuple_Size(new_base) - 1; } + else { + PyErr_SetString(PyExc_TypeError, + "__mro_entry__ must return a tuple"); + Py_DECREF(meth); + return NULL; + } Py_DECREF(base); args[i] = new_base; *modified_bases = 1; From 7cc8d8ff0b1f54bbb4ecb131098e74afedf95032 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 14 Nov 2017 10:41:30 +0100 Subject: [PATCH 41/82] Rename to __mro_entries__ --- Lib/test/test_genericclass.py | 22 +++++++++++----------- Lib/test/test_types.py | 19 ++++++++++++++----- Lib/types.py | 6 +++--- Objects/typeobject.c | 2 +- Python/bltinmodule.c | 6 +++--- 5 files changed, 32 insertions(+), 23 deletions(-) diff --git a/Lib/test/test_genericclass.py b/Lib/test/test_genericclass.py index e810887b744094..214527b01fa871 100644 --- a/Lib/test/test_genericclass.py +++ b/Lib/test/test_genericclass.py @@ -6,7 +6,7 @@ def test_mro_entry_signature(self): tested = [] class B: ... class C: - def __mro_entry__(self, *args, **kwargs): + def __mro_entries__(self, *args, **kwargs): tested.extend([args, kwargs]) return (C,) c = C() @@ -20,7 +20,7 @@ def test_mro_entry(self): class A: ... class B: ... class C: - def __mro_entry__(self, bases): + def __mro_entries__(self, bases): tested.append(bases) return (self.__class__,) c = C() @@ -40,7 +40,7 @@ def test_mro_entry_none(self): class A: ... class B: ... class C: - def __mro_entry__(self, bases): + def __mro_entries__(self, bases): tested.append(bases) return () c = C() @@ -60,7 +60,7 @@ def test_mro_entry_with_builtins(self): tested = [] class A: ... class C: - def __mro_entry__(self, bases): + def __mro_entries__(self, bases): tested.append(bases) return (dict,) c = C() @@ -74,7 +74,7 @@ class D(A, c): ... def test_mro_entry_with_builtins_2(self): tested = [] class C: - def __mro_entry__(self, bases): + def __mro_entries__(self, bases): tested.append(bases) return (C,) c = C() @@ -87,13 +87,13 @@ class D(c, dict): ... def test_mro_entry_errors(self): class C_too_many: - def __mro_entry__(self, bases, something, other): + def __mro_entries__(self, bases, something, other): return () c = C_too_many() with self.assertRaises(TypeError): class D(c): ... class C_too_few: - def __mro_entry__(self): + def __mro_entries__(self): return () d = C_too_few() with self.assertRaises(TypeError): @@ -101,12 +101,12 @@ class D(d): ... def test_mro_entry_errors_2(self): class C_not_callable: - __mro_entry__ = "Surprise!" + __mro_entries__ = "Surprise!" c = C_not_callable() with self.assertRaises(TypeError): class D(c): ... class C_not_tuple: - def __mro_entry__(self): + def __mro_entries__(self): return object c = C_not_tuple() with self.assertRaises(TypeError): @@ -120,7 +120,7 @@ def __new__(mcls, name, bases, ns): return super().__new__(mcls, name, bases, ns) class A: ... class C: - def __mro_entry__(self, bases): + def __mro_entries__(self, bases): return (A,) c = C() class D(c, metaclass=Meta): @@ -137,7 +137,7 @@ class D(c, metaclass=Meta): def test_mro_entry_type_call(self): # Substitution should _not_ happen in direct type call class C: - def __mro_entry__(self, bases): + def __mro_entries__(self, bases): return () c = C() with self.assertRaisesRegex(TypeError, diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index b7dd83f50de44d..e822ff7bb5eeb1 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -847,8 +847,8 @@ def func(ns): def test_new_class_with_mro_entry(self): class A: pass class C: - def __mro_entry__(self, bases): - return A + def __mro_entries__(self, bases): + return (A,) c = C() D = types.new_class('D', (c,), {}) self.assertEqual(D.__bases__, (A,)) @@ -859,7 +859,7 @@ def test_new_class_with_mro_entry_none(self): class A: pass class B: pass class C: - def __mro_entry__(self, bases): + def __mro_entries__(self, bases): return () c = C() D = types.new_class('D', (A, c, B), {}) @@ -867,6 +867,15 @@ def __mro_entry__(self, bases): self.assertEqual(D.__orig_bases__, (A, c, B)) self.assertEqual(D.__mro__, (D, A, B, object)) + def test_new_class_with_mro_entry_error(self): + class A: pass + class C: + def __mro_entries__(self, bases): + return A + c = C() + with self.assertRaises(TypeError): + types.new_class('D', (c,), {}) + # Many of the following tests are derived from test_descr.py def test_prepare_class(self): # Basic test of metaclass derivation @@ -913,10 +922,10 @@ def test_resolve_bases(self): class A: pass class B: pass class C: - def __mro_entry__(self, bases): + def __mro_entries__(self, bases): if A in bases: return () - return A + return (A,) c = C() self.assertEqual(types.resolve_bases(()), ()) self.assertEqual(types.resolve_bases((c,)), (A,)) diff --git a/Lib/types.py b/Lib/types.py index ef97da2df02cd1..08aabd6343db60 100644 --- a/Lib/types.py +++ b/Lib/types.py @@ -75,12 +75,12 @@ def resolve_bases(bases): for i, base in enumerate(bases): if isinstance(base, type): continue - if not hasattr(base, "__mro_entry__"): + if not hasattr(base, "__mro_entries__"): continue - new_base = base.__mro_entry__(bases) + new_base = base.__mro_entries__(bases) updated = True if not isinstance(new_base, tuple): - new_bases[i] = new_base + raise TypeError("__mro_entries__ must return a tuple") else: new_bases[i:i+1] = new_base if not updated: diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 8d48bd845df5f8..4713082e19dac4 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -2379,7 +2379,7 @@ type_new(PyTypeObject *metatype, PyObject *args, PyObject *kwds) else { for (i = 0; i < nbases; i++) { tmp = PyTuple_GET_ITEM(bases, i); - if (!PyType_Check(tmp) && PyObject_GetAttrString(tmp, "__mro_entry__")) { + if (!PyType_Check(tmp) && PyObject_GetAttrString(tmp, "__mro_entries__")) { PyErr_SetString(PyExc_TypeError, "type() doesn't support MRO entry resolution; " "use types.new_class()"); diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 6b3f28f7fe1256..f282abee773ca1 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -61,7 +61,7 @@ update_bases(PyObject* bases, PyObject** args, int nargs, int* modified_bases) if (PyType_Check(base)) { continue; } - if (!(meth = PyObject_GetAttrString(base, "__mro_entry__"))) { + if (!(meth = PyObject_GetAttrString(base, "__mro_entries__"))) { if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { return NULL; } @@ -70,7 +70,7 @@ update_bases(PyObject* bases, PyObject** args, int nargs, int* modified_bases) } if (!PyCallable_Check(meth)) { PyErr_SetString(PyExc_TypeError, - "__mro_entry__ must be callable"); + "__mro_entries__ must be callable"); Py_DECREF(meth); return NULL; } @@ -83,7 +83,7 @@ update_bases(PyObject* bases, PyObject** args, int nargs, int* modified_bases) } else { PyErr_SetString(PyExc_TypeError, - "__mro_entry__ must return a tuple"); + "__mro_entries__ must return a tuple"); Py_DECREF(meth); return NULL; } From 9f9bf2d2704eaae0490ef44648bc2b9eab129719 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 14 Nov 2017 10:57:55 +0100 Subject: [PATCH 42/82] Use new name __mro_entries__; few more fixes --- Lib/test/test_typing.py | 9 +++------ Lib/typing.py | 14 +++++++++++--- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index c40e6d68f136c4..3afbd07cb83a19 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -766,10 +766,9 @@ def __len__(self): def test_multiple_bases(self): class MM1(MutableMapping[str, str], collections.abc.MutableMapping): pass - with self.assertRaises(TypeError): - # consistent MRO not possible - class MM2(collections.abc.MutableMapping, MutableMapping[str, str]): - pass + class MM2(collections.abc.MutableMapping, MutableMapping[str, str]): + pass + self.assertEqual(MM2.__bases__, (collections.abc.MutableMapping, Generic)) def test_orig_bases(self): T = TypeVar('T') @@ -1123,7 +1122,6 @@ class C(Generic[T]): 'GenericTests.test_repr_2..C') X = C[int] self.assertEqual(X.__module__, __name__) - self.assertTrue(X.__qualname__.endswith('..C')) self.assertEqual(repr(X).split('.')[-1], 'C[int]') class Y(C[int]): @@ -1132,7 +1130,6 @@ class Y(C[int]): self.assertEqual(Y.__module__, __name__) self.assertEqual(Y.__qualname__, 'GenericTests.test_repr_2..Y') - self.assertEqual(repr(Y).split('.')[-1], 'Y') def test_eq_1(self): self.assertEqual(Generic, Generic) diff --git a/Lib/typing.py b/Lib/typing.py index ac0868e393e5f8..e2ea5c5243ef29 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -630,6 +630,8 @@ def __init__(self, origin, params, *, name=None, subcls=True, inst=True, special () if a is _TypingEmpty else a for a in params) self.__parameters__ = _type_vars(params) + if not name: + self.__module__ = origin.__module__ def _get_type_vars(self, tvars): if not self._special: @@ -695,15 +697,21 @@ def __call__(self, *args, **kwargs): pass return result - def __mro_entry__(self, bases): + def __mro_entries__(self, bases): if self._name: - return (self.__origin__, Generic) + res = [] + if self.__origin__ not in bases: + res.append(self.__origin__) + i = bases.index(self) + if not any(isinstance(b, _GenericAlias) for b in bases[i+1:]): + res.append(Generic) + return tuple(res) if self.__origin__ is Generic: i = bases.index(self) for b in bases[i+1:]: if isinstance(b, _GenericAlias): return () - return self.__origin__ + return (self.__origin__,) def __getattr__(self, attr): if '__origin__' in self.__dict__ and not _is_dunder(attr): # We are carefull for copy and pickle From d50ffa131f07d9c6b0686540cce5d57f1dc250d0 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 14 Nov 2017 12:24:00 +0100 Subject: [PATCH 43/82] Simplify class structure --- Lib/test/test_typing.py | 21 +- Lib/typing.py | 431 +++++++++++++--------------------------- 2 files changed, 150 insertions(+), 302 deletions(-) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 3afbd07cb83a19..aaf97f501c3d54 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -399,7 +399,7 @@ def test_tuple_instance_type_error(self): self.assertIsInstance((0, 0), Tuple) def test_repr(self): - self.assertEqual(repr(Tuple), "") + self.assertEqual(repr(Tuple), 'typing.Tuple') self.assertEqual(repr(Tuple[()]), 'typing.Tuple[()]') self.assertEqual(repr(Tuple[int, float]), 'typing.Tuple[int, float]') self.assertEqual(repr(Tuple[int, ...]), 'typing.Tuple[int, ...]') @@ -2432,15 +2432,8 @@ def test_alias_equality(self): self.assertNotEqual(Pattern[str], str) def test_errors(self): - with self.assertRaises(TypeError): - # Doesn't fit AnyStr. - Pattern[int] - with self.assertRaises(TypeError): - # Can't change type vars? - Match[T] m = Match[Union[str, bytes]] with self.assertRaises(TypeError): - # Too complicated? m[str] with self.assertRaises(TypeError): # We don't support isinstance(). @@ -2450,12 +2443,12 @@ def test_errors(self): issubclass(Pattern[bytes], Pattern[str]) def test_repr(self): - self.assertEqual(repr(Pattern), 'Pattern[~AnyStr]') - self.assertEqual(repr(Pattern[str]), 'Pattern[str]') - self.assertEqual(repr(Pattern[bytes]), 'Pattern[bytes]') - self.assertEqual(repr(Match), 'Match[~AnyStr]') - self.assertEqual(repr(Match[str]), 'Match[str]') - self.assertEqual(repr(Match[bytes]), 'Match[bytes]') + self.assertEqual(repr(Pattern), 'typing.Pattern') + self.assertEqual(repr(Pattern[str]), 'typing.Pattern[str]') + self.assertEqual(repr(Pattern[bytes]), 'typing.Pattern[bytes]') + self.assertEqual(repr(Match), 'typing.Match') + self.assertEqual(repr(Match[str]), 'typing.Match[str]') + self.assertEqual(repr(Match[bytes]), 'typing.Match[bytes]') def test_re_submodule(self): from typing.re import Match, Pattern, __all__, __name__ diff --git a/Lib/typing.py b/Lib/typing.py index e2ea5c5243ef29..8d22598a7cb54b 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -93,7 +93,7 @@ def _trim_name(nm): def _get_type_vars(types, tvars): for t in types: - if isinstance(t, _TypingBase): + if isinstance(t, (_GenericAlias, TypeVar)): t._get_type_vars(tvars) @@ -104,18 +104,13 @@ def _type_vars(types): def _eval_type(t, globalns, localns): - if isinstance(t, _TypingBase): + if isinstance(t, (_GenericAlias, _ForwardRef)): return t._eval_type(globalns, localns) return t - -Generic = object() _GenericAlias = None +Generic = object() _Protocol = object() -ClassVar = object() -Union = object() -NoReturn = object() -Optional = object() def _type_check(arg, msg): """Check that the argument is a type, and return it (internal helper). @@ -129,6 +124,8 @@ def _type_check(arg, msg): We append the repr() of the actual value (truncated to 100 chars). """ + if isinstance(arg, (type, TypeVar, _ForwardRef)): + return arg if arg is None: return type(None) if isinstance(arg, str): @@ -250,11 +247,19 @@ def inner(*args, **kwds): return func(*args, **kwds) return inner +class _Final: + """Mixin to prohibit subclassing""" + __slots__ = () + + def __init_subclass__(self, *args, **kwds): + if '_root' not in kwds: + raise TypeError("Cannot subclass special typing classes") + -class _TypingBase: +class _SpecialForm(_Final, _root=True): """Internal indicator of special typing constructs.""" - __slots__ = ('__weakref__',) + __slots__ = ('__weakref__', '_name', '_doc') def __new__(cls, *args, **kwds): """Constructor. @@ -269,28 +274,15 @@ def __new__(cls, *args, **kwds): raise TypeError("Cannot subclass %r" % cls) return super().__new__(cls) - def _eval_type(self, globalns, localns): - return self - - def _get_type_vars(self, tvars): - pass + def __init__(self, name, doc): + self._name = name + self._doc = doc def __repr__(self): - cls = type(self) - qname = _trim_name(cls.__qualname__) - return '%s.%s' % (cls.__module__, qname) + return 'typing.' + self._name def __call__(self, *args, **kwds): - raise TypeError("Cannot instantiate %r" % type(self)) - - -class _FinalTypingBase(_TypingBase): - - __slots__ = () - - def __init_subclass__(self, *args, **kwds): - if not kwds.pop('_root', False): - raise TypeError("Cannot subclass special typing classes") + raise TypeError("Cannot instantiate %r" % self) def __instancecheck__(self, obj): raise TypeError("%r cannot be used with isinstance()." % self) @@ -298,27 +290,118 @@ def __instancecheck__(self, obj): def __subclasscheck__(self, cls): raise TypeError("%r cannot be used with issubclass()." % self) + @_tp_cache + def __getitem__(self, parameters): + if self._name == 'ClassVar': + item = _type_check(parameters, 'ClassVar accepts only single type.') + return _GenericAlias(self, (item,)) + if self._name == 'Union': + if parameters == (): + raise TypeError("Cannot take a Union of no types.") + if not isinstance(parameters, tuple): + parameters = (parameters,) + msg = "Union[arg, ...]: each arg must be a type." + parameters = tuple(_type_check(p, msg) for p in parameters) + parameters = _remove_dups_flatten(parameters) + if len(parameters) == 1: + return parameters[0] + return _GenericAlias(self, parameters) + if self._name == 'Optional': + arg = _type_check(parameters, "Optional[t] requires a single type.") + return Union[arg, type(None)] + raise TypeError(f"{self} is not subscriptable") + + +Any = _SpecialForm('Any', doc= + """Special type indicating an unconstrained type. -class _SingletonTypingBase(_FinalTypingBase, _root=True): - """Internal mix-in class to prevent instantiation. + - Any is compatible with every type. + - Any assumed to have all methods. + - All values assumed to be instances of Any. - Prevents instantiation unless _root=True is given in class call. - It is used to create pseudo-singleton instances Any, Union, Optional, etc. - """ + Note that all the above statements are true from the point of view of + static type checkers. At runtime, Any should not be used with instance + or class checks. + """) - __slots__ = () +NoReturn = _SpecialForm('NoReturn', doc= + """Special type indicating functions that never return. + Example:: - def __new__(cls, *args, _root=False, **kwds): - self = super().__new__(cls, *args, **kwds) - if _root is True: - return self - raise TypeError("Cannot instantiate %r" % cls) + from typing import NoReturn + + def stop() -> NoReturn: + raise Exception('no way') + + This type is invalid in other positions, e.g., ``List[NoReturn]`` + will fail in static type checkers. + """) + +ClassVar = _SpecialForm('ClassVar', doc= + """Special type construct to mark class variables. + + An annotation wrapped in ClassVar indicates that a given + attribute is intended to be used as a class variable and + should not be set on instances of that class. Usage:: + + class Starship: + stats: ClassVar[Dict[str, int]] = {} # class variable + damage: int = 10 # instance variable + + ClassVar accepts only types and cannot be further subscribed. + + Note that ClassVar is not a class itself, and should not + be used with isinstance() or issubclass(). + """) + +Union = _SpecialForm('Union', doc= + """Union type; Union[X, Y] means either X or Y. + + To define a union, use e.g. Union[int, str]. Details: + - The arguments must be types and there must be at least one. + - None as an argument is a special case and is replaced by + type(None). + - Unions of unions are flattened, e.g.:: + + Union[Union[int, str], float] == Union[int, str, float] + + - Unions of a single argument vanish, e.g.:: + + Union[int] == int # The constructor actually returns int + + - Redundant arguments are skipped, e.g.:: + + Union[int, str, int] == Union[int, str] + + - When comparing unions, the argument order is ignored, e.g.:: + + Union[int, str] == Union[str, int] + + - When two arguments have a subclass relationship, the least + derived argument is kept, e.g.:: + + class Employee: pass + class Manager(Employee): pass + Union[int, Employee, Manager] == Union[int, Employee] + Union[Manager, int, Employee] == Union[int, Employee] + Union[Employee, Manager] == Employee + + - Similar for object:: + + Union[int, object] == object + + - You cannot subclass or instantiate a union. + - You can use Optional[X] as a shorthand for Union[X, None]. + """) + +Optional = _SpecialForm('Optional', doc= + """Optional type. - def __reduce__(self): - return _trim_name(type(self).__name__) + Optional[X] is equivalent to Union[X, None]. + """) -class _ForwardRef(_FinalTypingBase, _root=True): +class _ForwardRef(_Final, _root=True): """Internal wrapper to hold a forward reference.""" __slots__ = ('__forward_arg__', '__forward_code__', @@ -364,7 +447,7 @@ def __repr__(self): return '_ForwardRef(%r)' % (self.__forward_arg__,) -class TypeVar(_FinalTypingBase, _root=True): +class TypeVar(_Final, _root=True): """Type variable. Usage:: @@ -455,152 +538,6 @@ def __repr__(self): # (This one *is* for export!) AnyStr = TypeVar('AnyStr', bytes, str) - -class _Any(_SingletonTypingBase, _root=True): - """Special type indicating an unconstrained type. - - - Any is compatible with every type. - - Any assumed to have all methods. - - All values assumed to be instances of Any. - - Note that all the above statements are true from the point of view of - static type checkers. At runtime, Any should not be used with instance - or class checks. - """ - - __slots__ = () - - -Any = _Any(_root=True) - - -class _NoReturn(_SingletonTypingBase, _root=True): - """Special type indicating functions that never return. - Example:: - - from typing import NoReturn - - def stop() -> NoReturn: - raise Exception('no way') - - This type is invalid in other positions, e.g., ``List[NoReturn]`` - will fail in static type checkers. - """ - - __slots__ = () - - -NoReturn = _NoReturn(_root=True) - - -class _ClassVar(_SingletonTypingBase, _root=True): - """Special type construct to mark class variables. - - An annotation wrapped in ClassVar indicates that a given - attribute is intended to be used as a class variable and - should not be set on instances of that class. Usage:: - - class Starship: - stats: ClassVar[Dict[str, int]] = {} # class variable - damage: int = 10 # instance variable - - ClassVar accepts only types and cannot be further subscribed. - - Note that ClassVar is not a class itself, and should not - be used with isinstance() or issubclass(). - """ - - __slots__ = () - - @_tp_cache - def __getitem__(self, item): - item = _type_check(item, 'ClassVar accepts only single type.') - return _GenericAlias(self, (item,)) - - -ClassVar = _ClassVar(_root=True) - - -class _Union(_SingletonTypingBase, _root=True): - """Union type; Union[X, Y] means either X or Y. - - To define a union, use e.g. Union[int, str]. Details: - - - The arguments must be types and there must be at least one. - - - None as an argument is a special case and is replaced by - type(None). - - - Unions of unions are flattened, e.g.:: - - Union[Union[int, str], float] == Union[int, str, float] - - - Unions of a single argument vanish, e.g.:: - - Union[int] == int # The constructor actually returns int - - - Redundant arguments are skipped, e.g.:: - - Union[int, str, int] == Union[int, str] - - - When comparing unions, the argument order is ignored, e.g.:: - - Union[int, str] == Union[str, int] - - - When two arguments have a subclass relationship, the least - derived argument is kept, e.g.:: - - class Employee: pass - class Manager(Employee): pass - Union[int, Employee, Manager] == Union[int, Employee] - Union[Manager, int, Employee] == Union[int, Employee] - Union[Employee, Manager] == Employee - - - Similar for object:: - - Union[int, object] == object - - - You cannot subclass or instantiate a union. - - - You can use Optional[X] as a shorthand for Union[X, None]. - """ - - __slots__ = () - - @_tp_cache - def __getitem__(self, parameters): - if parameters == (): - raise TypeError("Cannot take a Union of no types.") - if not isinstance(parameters, tuple): - parameters = (parameters,) - msg = "Union[arg, ...]: each arg must be a type." - parameters = tuple(_type_check(p, msg) for p in parameters) - parameters = _remove_dups_flatten(parameters) - if len(parameters) == 1: - return parameters[0] - return _GenericAlias(self, parameters) - - -Union = _Union(_root=True) - - -class _Optional(_SingletonTypingBase, _root=True): - """Optional type. - - Optional[X] is equivalent to Union[X, None]. - """ - - __slots__ = () - - @_tp_cache - def __getitem__(self, arg): - arg = _type_check(arg, "Optional[t] requires a single type.") - return Union[arg, type(None)] - - -Optional = _Optional(_root=True) - - # Special typing constructs Union, Optional, Generic, Callable and Tuple # use three special attributes for internal bookkeeping of generic types: # * __parameters__ is a tuple of unique free type parameters of a generic @@ -616,7 +553,7 @@ def _is_dunder(attr): return attr.startswith('__') and attr.endswith('__') -class _GenericAlias(_FinalTypingBase, _root=True): +class _GenericAlias(_Final, _root=True): def __init__(self, origin, params, *, name=None, subcls=True, inst=True, special=False): self._name = name @@ -649,6 +586,18 @@ def __getitem__(self, params): if self.__origin__ in (Generic, _Protocol): # Can't subscript Generic[...] or _Protocol[...]. raise TypeError("Cannot subscript already-subscripted {self}") + if self.__origin__ is tuple and self._special: + if params == (): + return _GenericAlias(tuple, (_TypingEmpty,), name=self._name, inst=self._inst, subcls=self._subcls) + if not isinstance(params, tuple): + params = (params,) + if len(params) == 2 and params[1] is ...: + msg = "Tuple[t, ...]: t must be a type." + p = _type_check(params[0], msg) + return _GenericAlias(tuple, (p, _TypingEllipsis), name=self._name, inst=self._inst, subcls=self._subcls) + msg = "Tuple[t0, t1, ...]: each t must be a type." + params = tuple(_type_check(p, msg) for p in params) + return _GenericAlias(tuple, params, name=self._name, inst=self._inst, subcls=self._subcls) if not isinstance(params, tuple): params = (params,) msg = "Parameters to generic types must be types." @@ -870,7 +819,8 @@ class _TypingEllipsis: """Internal placeholder for ... (ellipsis).""" -class Tuple(tuple, Generic, metaclass=abc.ABCMeta): +Tuple = _GenericAlias(tuple, (), name='Tuple', inst=False, special=True) +if False: """Tuple type; Tuple[X, Y] is the cross-product type of X and Y. Example: Tuple[T1, T2] is a tuple of two elements corresponding @@ -880,33 +830,6 @@ class Tuple(tuple, Generic, metaclass=abc.ABCMeta): To specify a variable-length tuple of homogeneous type, use Tuple[T, ...]. """ - __slots__ = () - __call__ = None - - def __new__(cls, *args, **kwds): - if cls is Tuple: - raise TypeError("Type Tuple cannot be instantiated; " - "use tuple() instead") - return super().__new__(cls, *args, **kwds) - - @_tp_cache - def __class_getitem__(cls, parameters): - if cls is not Tuple: - # Normal generic rules apply if this is not the first subscription - # or a subscription of a subclass. - return super().__class_getitem__(cls, parameters) - if parameters == (): - return super().__class_getitem__(cls, (_TypingEmpty,)) - if not isinstance(parameters, tuple): - parameters = (parameters,) - if len(parameters) == 2 and parameters[1] is ...: - msg = "Tuple[t, ...]: t must be a type." - p = _type_check(parameters[0], msg) - return super().__class_getitem__(cls, (p, _TypingEllipsis)) - msg = "Tuple[t0, t1, ...]: each t must be a type." - parameters = tuple(_type_check(p, msg) for p in parameters) - return super().__class_getitem__(cls, parameters) - class Callable(collections.abc.Callable, Generic): """Callable type; Callable[[int], str] is a function of (int) -> str. @@ -1647,76 +1570,8 @@ class io: io.__name__ = __name__ + '.io' sys.modules[io.__name__] = io - -class _TypeAlias(_FinalTypingBase, _root=True): - """Internal helper class for defining generic variants of concrete types. - - Note that this is not a type; let's call it a pseudo-type. It cannot - be used in instance and subclass checks in parameterized form, i.e. - ``isinstance(42, Match[str])`` raises ``TypeError`` instead of returning - ``False``. - """ - - __slots__ = ('name', 'type_var', 'impl_type', 'type_checker') - - def __init__(self, name, type_var, impl_type, type_checker): - """Initializer. - - Args: - name: The name, e.g. 'Pattern'. - type_var: The type parameter, e.g. AnyStr, or the - specific type, e.g. str. - impl_type: The implementation type. - type_checker: Function that takes an impl_type instance. - and returns a value that should be a type_var instance. - """ - assert isinstance(name, str), repr(name) - assert isinstance(impl_type, type), repr(impl_type) - assert isinstance(type_var, (type, _TypingBase)), repr(type_var) - self.name = name - self.type_var = type_var - self.impl_type = impl_type - self.type_checker = type_checker - - def __repr__(self): - return f"{self.name}[{_type_repr(self.type_var)}]" - - def __getitem__(self, parameter): - if not isinstance(self.type_var, TypeVar): - raise TypeError(f"{self} cannot be further parameterized.") - if self.type_var.__constraints__ and isinstance(parameter, type): - if not issubclass(parameter, self.type_var.__constraints__): - raise TypeError(f"{parameter} is not a valid substitution for {self.type_var}.") - if isinstance(parameter, TypeVar) and parameter is not self.type_var: - raise TypeError(f"{self} cannot be re-parameterized.") - return self.__class__(self.name, parameter, - self.impl_type, self.type_checker) - - def __eq__(self, other): - if not isinstance(other, _TypeAlias): - return NotImplemented - return self.name == other.name and self.type_var == other.type_var - - def __hash__(self): - return hash((self.name, self.type_var)) - - def __instancecheck__(self, obj): - if not isinstance(self.type_var, TypeVar): - raise TypeError("Parameterized type aliases cannot be used " - "with isinstance().") - return isinstance(obj, self.impl_type) - - def __subclasscheck__(self, cls): - if not isinstance(self.type_var, TypeVar): - raise TypeError("Parameterized type aliases cannot be used " - "with issubclass().") - return issubclass(cls, self.impl_type) - - -Pattern = _TypeAlias('Pattern', AnyStr, type(stdlib_re.compile('')), - lambda p: p.pattern) -Match = _TypeAlias('Match', AnyStr, type(stdlib_re.match('', '')), - lambda m: m.re.pattern) +Pattern = _GenericAlias(type(stdlib_re.compile('')), AnyStr, name='Pattern', subcls=False, special=True) +Match = _GenericAlias(type(stdlib_re.match('', '')), AnyStr, name='Match', subcls=False, special=True) class re: """Wrapper namespace for re type aliases.""" From cac30d4b9f58e3240f0cc9969afaf3c2cfe4135b Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 14 Nov 2017 12:52:12 +0100 Subject: [PATCH 44/82] Use _GenericAlias also for Callable --- Lib/typing.py | 89 +++++++++++++++++++++------------------------------ 1 file changed, 37 insertions(+), 52 deletions(-) diff --git a/Lib/typing.py b/Lib/typing.py index 8d22598a7cb54b..e1120781ece03d 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -84,12 +84,6 @@ # namespace, but excluded from __all__ because they might stomp on # legitimate imports of those modules. -def _trim_name(nm): - whitelist = ('_TypingBase', '_FinalTypingBase', '_SingletonTypingBase', '_ForwardRef') - if nm.startswith('_') and nm not in whitelist: - nm = nm[1:] - return nm - def _get_type_vars(types, tvars): for t in types: @@ -581,8 +575,23 @@ def _eval_type(self, globalns, localns): return _GenericAlias(self.__origin__, ev_args, name=self._name, subcls=self._subcls, inst=self._inst, special=self._special) - @_tp_cache def __getitem__(self, params): + if self._name != 'Callable' or not self._special: + return self.__getitem_inner__(params) + if not isinstance(params, tuple) or len(params) != 2: + raise TypeError("Callable must be used as " + "Callable[[arg, ...], result].") + args, result = params + if args is Ellipsis: + params = (Ellipsis, result) + else: + if not isinstance(args, list): + raise TypeError(f"Callable[args, result]: args must be a list. Got {args}") + params = (tuple(args), result) + return self.__getitem_inner__(params) + + @_tp_cache + def __getitem_inner__(self, params): if self.__origin__ in (Generic, _Protocol): # Can't subscript Generic[...] or _Protocol[...]. raise TypeError("Cannot subscript already-subscripted {self}") @@ -598,6 +607,17 @@ def __getitem__(self, params): msg = "Tuple[t0, t1, ...]: each t must be a type." params = tuple(_type_check(p, msg) for p in params) return _GenericAlias(tuple, params, name=self._name, inst=self._inst, subcls=self._subcls) + if self.__origin__ is collections.abc.Callable and self._special: + args, result = params + msg = "Callable[args, result]: result must be a type." + result = _type_check(result, msg) + if args is Ellipsis: + return _GenericAlias(self.__origin__, (_TypingEllipsis, result), + name=self._name, inst=self._inst, subcls=self._subcls) + msg = "Callable[[arg, ...], result]: each arg must be a type." + args = tuple(_type_check(arg, msg) for arg in args) + params = args + (result,) + return _GenericAlias(self.__origin__, params, name=self._name, inst=self._inst, subcls=self._subcls) if not isinstance(params, tuple): params = (params,) msg = "Parameters to generic types must be types." @@ -606,7 +626,7 @@ def __getitem__(self, params): return _subs_tvars(self, self.__parameters__, params) def __repr__(self): - if (self.__origin__ is not Callable or + if (self._name != 'Callable' or len(self.__args__) == 2 and self.__args__[0] is Ellipsis): if self._name: name = 'typing.' + self._name @@ -617,6 +637,8 @@ def __repr__(self): else: args = '' return (f'{name}{args}') + if self._special: + return 'typing.Callable' return (f'typing.Callable' f'[[{", ".join([_type_repr(a) for a in self.__args__[:-1]])}], ' f'{_type_repr(self.__args__[-1])}]') @@ -677,9 +699,11 @@ def __instancecheck__(self, obj): def __subclasscheck__(self, cls): if self._special: - return issubclass(cls, self.__origin__) - else: - raise TypeError("Subscripted generic cannot be used with class and instance checks") + if not isinstance(cls, _GenericAlias): + return issubclass(cls, self.__origin__) + if cls._special: + return issubclass(cls.__origin__, self.__origin__) + raise TypeError("Subscripted generics cannot be used with class and instance checks") def _make_subclasshook(cls): @@ -831,7 +855,8 @@ class _TypingEllipsis: """ -class Callable(collections.abc.Callable, Generic): +Callable = _GenericAlias(collections.abc.Callable, (), name='Callable', special=True) +if False: """Callable type; Callable[[int], str] is a function of (int) -> str. The subscription syntax must always be used with exactly two @@ -842,46 +867,6 @@ class Callable(collections.abc.Callable, Generic): such function types are rarely used as callback types. """ - __slots__ = () - - def __new__(cls, *args, **kwds): - if cls is Callable: - raise TypeError("Type Callable cannot be instantiated; " - "use a non-abstract subclass instead") - return super().__new__(cls, *args, **kwds) - - def __class_getitem__(cls, parameters): - """A thin wrapper around __getitem_inner__ to provide the latter - with hashable arguments to improve speed. - """ - - if cls is not Callable: - return super().__class_getitem__(cls, parameters) - if not isinstance(parameters, tuple) or len(parameters) != 2: - raise TypeError("Callable must be used as " - "Callable[[arg, ...], result].") - args, result = parameters - if args is Ellipsis: - parameters = (Ellipsis, result) - else: - if not isinstance(args, list): - raise TypeError(f"Callable[args, result]: args must be a list. Got {args}") - parameters = (tuple(args), result) - return cls.__getitem_inner__(parameters) - - @classmethod - @_tp_cache - def __getitem_inner__(cls, parameters): - args, result = parameters - msg = "Callable[args, result]: result must be a type." - result = _type_check(result, msg) - if args is Ellipsis: - return super().__class_getitem__(cls, (_TypingEllipsis, result)) - msg = "Callable[[arg, ...], result]: each arg must be a type." - args = tuple(_type_check(arg, msg) for arg in args) - parameters = args + (result,) - return super().__class_getitem__(cls, parameters) - def cast(typ, val): """Cast a value to a type. From 3a608b7eb8ef7883d41f09e6e8e2fe06bfa01fdf Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 14 Nov 2017 12:57:10 +0100 Subject: [PATCH 45/82] Reorganize code --- Lib/typing.py | 48 ++++++++++++++++++++++-------------------------- 1 file changed, 22 insertions(+), 26 deletions(-) diff --git a/Lib/typing.py b/Lib/typing.py index e1120781ece03d..8473c11c2a9cdb 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -241,6 +241,7 @@ def inner(*args, **kwds): return func(*args, **kwds) return inner + class _Final: """Mixin to prohibit subclassing""" __slots__ = () @@ -843,31 +844,6 @@ class _TypingEllipsis: """Internal placeholder for ... (ellipsis).""" -Tuple = _GenericAlias(tuple, (), name='Tuple', inst=False, special=True) -if False: - """Tuple type; Tuple[X, Y] is the cross-product type of X and Y. - - Example: Tuple[T1, T2] is a tuple of two elements corresponding - to type variables T1 and T2. Tuple[int, float, str] is a tuple - of an int, a float and a string. - - To specify a variable-length tuple of homogeneous type, use Tuple[T, ...]. - """ - - -Callable = _GenericAlias(collections.abc.Callable, (), name='Callable', special=True) -if False: - """Callable type; Callable[[int], str] is a function of (int) -> str. - - The subscription syntax must always be used with exactly two - values: the argument list and the return type. The argument list - must be a list of types or ellipsis; the return type must be a single type. - - There is no syntax to indicate optional or keyword arguments, - such function types are rarely used as callback types. - """ - - def cast(typ, val): """Cast a value to a type. @@ -1212,7 +1188,17 @@ def __round__(self, ndigits: int = 0) -> T_co: Sized = _GenericAlias(collections.abc.Sized, (), name='Sized', special=True) # Not generic. Container = _GenericAlias(collections.abc.Container, T_co, name='Container', special=True) Collection = _GenericAlias(collections.abc.Collection, T_co, name='Collection', special=True) -# Callable was defined earlier. +Callable = _GenericAlias(collections.abc.Callable, (), name='Callable', special=True) +Callable.__doc__ = \ + """Callable type; Callable[[int], str] is a function of (int) -> str. + + The subscription syntax must always be used with exactly two + values: the argument list and the return type. The argument list + must be a list of types or ellipsis; the return type must be a single type. + + There is no syntax to indicate optional or keyword arguments, + such function types are rarely used as callback types. + """ AbstractSet = _GenericAlias(collections.abc.Set, T_co, name='AbstractSet', special=True) MutableSet = _GenericAlias(collections.abc.MutableSet, T, name='MutableSet', special=True) # NOTE: Mapping is only covariant in the value type. @@ -1222,6 +1208,16 @@ def __round__(self, ndigits: int = 0) -> T_co: MutableSequence = _GenericAlias(collections.abc.MutableSequence, T, name='MutableSequence', special=True) # Not generic ByteString = _GenericAlias(collections.abc.ByteString, (), name='ByteString', special=True) +Tuple = _GenericAlias(tuple, (), name='Tuple', inst=False, special=True) +Tuple.__doc__ = \ + """Tuple type; Tuple[X, Y] is the cross-product type of X and Y. + + Example: Tuple[T1, T2] is a tuple of two elements corresponding + to type variables T1 and T2. Tuple[int, float, str] is a tuple + of an int, a float and a string. + + To specify a variable-length tuple of homogeneous type, use Tuple[T, ...]. + """ List = _GenericAlias(list, T, name='List', inst=False, special=True) Deque = _GenericAlias(collections.deque, T, name='Deque', special=True) Set = _GenericAlias(set, T, name='Set', inst=False, special=True) From 1492a4c91aa3b50dcb62ba7f19bf4bf476291bab Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 14 Nov 2017 13:01:05 +0100 Subject: [PATCH 46/82] Get rid of _make_subclasshook --- Lib/typing.py | 37 +------------------------------------ 1 file changed, 1 insertion(+), 36 deletions(-) diff --git a/Lib/typing.py b/Lib/typing.py index 8473c11c2a9cdb..369675a0c57d0d 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -707,40 +707,6 @@ def __subclasscheck__(self, cls): raise TypeError("Subscripted generics cannot be used with class and instance checks") -def _make_subclasshook(cls): - """Construct a __subclasshook__ callable that incorporates - the associated __extra__ class in subclass checks performed - against cls. - """ - if cls.__module__ != 'typing': - def __extrahook__(subclass): - return NotImplemented - return __extrahook__ - extra = cls.__bases__[0] - if isinstance(extra, abc.ABCMeta): - # The logic mirrors that of ABCMeta.__subclasscheck__. - def __extrahook__(subclass): - res = extra.__subclasshook__(subclass) - if res is not NotImplemented: - return res - if extra in subclass.__mro__: - return True - for rcls in extra._abc_registry: - if rcls is not cls and issubclass(subclass, rcls): - return True - for scls in extra.__subclasses__(): - if scls is not cls and issubclass(subclass, scls): - return True - return NotImplemented - else: - # For non-ABC extras we'll just call issubclass(). - def __extrahook__(subclass): - if issubclass(subclass, extra): - return True - return NotImplemented - return __extrahook__ - - class Generic: """Abstract base class for generic types. @@ -761,6 +727,7 @@ def lookup_name(mapping: Mapping[KT, VT], key: KT, default: VT) -> VT: except KeyError: return default """ + __slots__ = () def __new__(cls, *args, **kwds): @@ -829,8 +796,6 @@ def __init_subclass__(cls, *args, **kwargs): "are not listed in Generic[{', '.join(str(g) for g in gvars)}]") tvars = gvars cls.__parameters__ = tuple(tvars) - if cls.__module__ == 'typing' or cls.__subclasshook__.__name__ == '__extrahook__': - cls.__subclasshook__ = _make_subclasshook(cls) class _TypingEmpty: From c0cd0ca4aab19b7e72e91796beb478ea63e19156 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 14 Nov 2017 13:03:57 +0100 Subject: [PATCH 47/82] Make ForwardRef public --- Lib/test/test_typing.py | 10 +++++----- Lib/typing.py | 16 ++++++++-------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index aaf97f501c3d54..8f9f46e07f06e4 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -1332,22 +1332,22 @@ def add_right(self, node: 'Node[T]' = None): self.assertEqual(right_hints['node'], Optional[Node[T]]) def test_forwardref_instance_type_error(self): - fr = typing._ForwardRef('int') + fr = typing.ForwardRef('int') with self.assertRaises(TypeError): isinstance(42, fr) def test_forwardref_subclass_type_error(self): - fr = typing._ForwardRef('int') + fr = typing.ForwardRef('int') with self.assertRaises(TypeError): issubclass(int, fr) def test_forward_equality(self): - fr = typing._ForwardRef('int') - self.assertEqual(fr, typing._ForwardRef('int')) + fr = typing.ForwardRef('int') + self.assertEqual(fr, typing.ForwardRef('int')) self.assertNotEqual(List['int'], List[int]) def test_forward_repr(self): - self.assertEqual(repr(List['int']), "typing.List[_ForwardRef('int')]") + self.assertEqual(repr(List['int']), "typing.List[ForwardRef('int')]") def test_union_forward(self): diff --git a/Lib/typing.py b/Lib/typing.py index 369675a0c57d0d..ebbc95d86b0a05 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -98,7 +98,7 @@ def _type_vars(types): def _eval_type(t, globalns, localns): - if isinstance(t, (_GenericAlias, _ForwardRef)): + if isinstance(t, (_GenericAlias, ForwardRef)): return t._eval_type(globalns, localns) return t @@ -118,12 +118,12 @@ def _type_check(arg, msg): We append the repr() of the actual value (truncated to 100 chars). """ - if isinstance(arg, (type, TypeVar, _ForwardRef)): + if isinstance(arg, (type, TypeVar, ForwardRef)): return arg if arg is None: return type(None) if isinstance(arg, str): - return _ForwardRef(arg) + return ForwardRef(arg) if ( # Bare Union etc. are not valid as type arguments _GenericAlias and isinstance(arg, _GenericAlias) and arg.__origin__ in (Generic, _Protocol, ClassVar) or @@ -396,7 +396,7 @@ class Manager(Employee): pass """) -class _ForwardRef(_Final, _root=True): +class ForwardRef(_Final, _root=True): """Internal wrapper to hold a forward reference.""" __slots__ = ('__forward_arg__', '__forward_code__', @@ -430,7 +430,7 @@ def _eval_type(self, globalns, localns): return self.__forward_value__ def __eq__(self, other): - if not isinstance(other, _ForwardRef): + if not isinstance(other, ForwardRef): return NotImplemented return (self.__forward_arg__ == other.__forward_arg__ and self.__forward_value__ == other.__forward_value__) @@ -439,7 +439,7 @@ def __hash__(self): return hash((self.__forward_arg__, self.__forward_value__)) def __repr__(self): - return '_ForwardRef(%r)' % (self.__forward_arg__,) + return 'ForwardRef(%r)' % (self.__forward_arg__,) class TypeVar(_Final, _root=True): @@ -891,7 +891,7 @@ def get_type_hints(obj, globalns=None, localns=None): if value is None: value = type(None) if isinstance(value, str): - value = _ForwardRef(value) + value = ForwardRef(value) value = _eval_type(value, base_globals, localns) hints[name] = value return hints @@ -919,7 +919,7 @@ def get_type_hints(obj, globalns=None, localns=None): if value is None: value = type(None) if isinstance(value, str): - value = _ForwardRef(value) + value = ForwardRef(value) value = _eval_type(value, globalns, localns) if name in defaults and defaults[name] is None: value = Optional[value] From 99d3ce90c52582080c0674e24db002f24cd284b6 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 14 Nov 2017 13:12:57 +0100 Subject: [PATCH 48/82] Some minor fixes --- Lib/test/test_typing.py | 11 ++--------- Lib/typing.py | 7 +++++-- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 8f9f46e07f06e4..1f190e2a9ccb1e 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -908,7 +908,7 @@ def test_extended_generic_rules_repr(self): self.assertEqual(repr(Union[Tuple, Callable]).replace('typing.', ''), 'Union[Tuple, Callable]') self.assertEqual(repr(Union[Tuple, Tuple[int]]).replace('typing.', ''), - "") + 'Union[Tuple, Tuple[int]]') self.assertEqual(repr(Callable[..., Optional[T]][int]).replace('typing.', ''), 'Callable[..., Union[int, NoneType]]') self.assertEqual(repr(Callable[[], List[T]][int]).replace('typing.', ''), @@ -1051,14 +1051,7 @@ class Node(Generic[T]): ... Union['T', int], List['T'], typing.Mapping['T', int]] for t in things + [Any]: self.assertEqual(t, copy(t)) - self.assertEqual(t, deepcopy(t)) - if sys.version_info >= (3, 3): - # From copy module documentation: - # It does "copy" functions and classes (shallow and deeply), by returning - # the original object unchanged; this is compatible with the way these - # are treated by the pickle module. - self.assertTrue(t is copy(t)) - self.assertTrue(t is deepcopy(t)) + self.assertEqual(repr(t), repr(deepcopy(t))) # Use repr() because of TypeVars def test_weakref_all(self): T = TypeVar('T') diff --git a/Lib/typing.py b/Lib/typing.py index ebbc95d86b0a05..77e848c64ae69e 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -118,8 +118,6 @@ def _type_check(arg, msg): We append the repr() of the actual value (truncated to 100 chars). """ - if isinstance(arg, (type, TypeVar, ForwardRef)): - return arg if arg is None: return type(None) if isinstance(arg, str): @@ -130,6 +128,8 @@ def _type_check(arg, msg): arg in (Generic, _Protocol, ClassVar, Union, NoReturn, Optional) ): raise TypeError("Plain %s is not valid as type argument" % arg) + if isinstance(arg, (type, TypeVar, ForwardRef)): + return arg if not callable(arg): raise TypeError(msg + " Got %.100r." % (arg,)) return arg @@ -276,6 +276,9 @@ def __init__(self, name, doc): def __repr__(self): return 'typing.' + self._name + def __copy__(self): + return self # Special forms are immutable. + def __call__(self, *args, **kwds): raise TypeError("Cannot instantiate %r" % self) From 68a0c1803b00334d01a6607ba23503f8f71ad036 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 14 Nov 2017 13:43:46 +0100 Subject: [PATCH 49/82] Fix slots test and remove irrelevant --- Lib/test/test_typing.py | 4 ---- Lib/typing.py | 27 ++++++++++++++++++++++++--- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 1f190e2a9ccb1e..42ae3ab85b538f 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -1068,7 +1068,6 @@ class C(Generic[T]): c = C() c_int = C[int]() - self.assertEqual(C.__slots__, C[str].__slots__) c.potato = 0 c_int.potato = 0 @@ -1079,8 +1078,6 @@ class C(Generic[T]): def foo(x: C['C']): ... self.assertEqual(get_type_hints(foo, globals(), locals())['x'], C[C]) - self.assertEqual(get_type_hints(foo, globals(), locals())['x'].__slots__, - C.__slots__) self.assertEqual(copy(C[int]), deepcopy(C[int])) def test_parameterized_slots_dict(self): @@ -1090,7 +1087,6 @@ class D(Generic[T]): d = D() d_int = D[int]() - self.assertEqual(D.__slots__, D[str].__slots__) d.banana = 'yes' d_int.banana = 'yes' diff --git a/Lib/typing.py b/Lib/typing.py index 77e848c64ae69e..a21e1ee09651b1 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -244,7 +244,6 @@ def inner(*args, **kwds): class _Final: """Mixin to prohibit subclassing""" - __slots__ = () def __init_subclass__(self, *args, **kwds): if '_root' not in kwds: @@ -254,8 +253,6 @@ def __init_subclass__(self, *args, **kwds): class _SpecialForm(_Final, _root=True): """Internal indicator of special typing constructs.""" - __slots__ = ('__weakref__', '_name', '_doc') - def __new__(cls, *args, **kwds): """Constructor. @@ -273,6 +270,14 @@ def __init__(self, name, doc): self._name = name self._doc = doc + def __eq__(self, other): + if not isinstance(other, _SpecialForm): + return NotImplemented + return self._name == other._name + + def __hash__(self): + return hash((self._name,)) + def __repr__(self): return 'typing.' + self._name @@ -490,6 +495,20 @@ def longest(x: A, y: A) -> A: __slots__ = ('__name__', '__bound__', '__constraints__', '__covariant__', '__contravariant__') + def __getstate__(self): + return {'name': self.__name__, + 'bound': self.__bound__, + 'constraints': self.__constraints__, + 'co': self.__covariant__, + 'contra': self.__contravariant__} + + def __setstate__(self, state): + self.__name__ = state['name'] + self.__bound__ = state['bound'] + self.__constraints__ = state['constraints'] + self.__covariant__ = state['co'] + self.__contravariant__ = state['contra'] + def __init__(self, name, *constraints, bound=None, covariant=False, contravariant=False): self.__name__ = name @@ -654,6 +673,8 @@ def __eq__(self, other): return False if self.__origin__ is Union and other.__origin__ is Union: return frozenset(self.__args__) == frozenset(other.__args__) + if self._special and other._special: + return True return self.__args__ == other.__args__ def __hash__(self): From 1f32259952bdae96b9fb16ed999bf240a763449d Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 14 Nov 2017 13:55:39 +0100 Subject: [PATCH 50/82] Fix _Protocol (temporary) --- Lib/test/test_typing.py | 2 +- Lib/typing.py | 102 ++++++++++++++++++++-------------------- 2 files changed, 52 insertions(+), 52 deletions(-) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 42ae3ab85b538f..aeaa7db3dc0ec6 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -682,7 +682,7 @@ def test_new_repr_complex(self): def test_new_repr_bare(self): T = TypeVar('T') self.assertEqual(repr(Generic[T]), 'typing.Generic[~T]') - self.assertEqual(repr(typing._Protocol[T]), 'typing.Protocol[~T]') + self.assertEqual(repr(typing._Protocol[T]), 'typing._Protocol[~T]') class C(typing.Dict[Any, Any]): ... # this line should just work repr(C.__mro__) diff --git a/Lib/typing.py b/Lib/typing.py index a21e1ee09651b1..c563dac2952c2e 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -1112,7 +1112,7 @@ class _Protocol(metaclass=_ProtocolMeta): _is_protocol = True def __class_getitem__(cls, params): - return cls + return Generic.__class_getitem__(cls, params) # Various ABCs mimicking those in collections.abc. @@ -1123,56 +1123,6 @@ def __class_getitem__(cls, params): AsyncIterator = _GenericAlias(collections.abc.AsyncIterator, T_co, name='AsyncIterator', special=True) Iterable = _GenericAlias(collections.abc.Iterable, T_co, name='Iterable', special=True) Iterator = _GenericAlias(collections.abc.Iterator, T_co, name='Iterator', special=True) - - -class SupportsInt(_Protocol): - __slots__ = () - - @abstractmethod - def __int__(self) -> int: - pass - - -class SupportsFloat(_Protocol): - __slots__ = () - - @abstractmethod - def __float__(self) -> float: - pass - - -class SupportsComplex(_Protocol): - __slots__ = () - - @abstractmethod - def __complex__(self) -> complex: - pass - - -class SupportsBytes(_Protocol): - __slots__ = () - - @abstractmethod - def __bytes__(self) -> bytes: - pass - - -class SupportsAbs(_Protocol[T_co]): - __slots__ = () - - @abstractmethod - def __abs__(self) -> T_co: - pass - - -class SupportsRound(_Protocol[T_co]): - __slots__ = () - - @abstractmethod - def __round__(self, ndigits: int = 0) -> T_co: - pass - - Reversible = _GenericAlias(collections.abc.Reversible, T_co, name='Reversible', special=True) Sized = _GenericAlias(collections.abc.Sized, (), name='Sized', special=True) # Not generic. Container = _GenericAlias(collections.abc.Container, T_co, name='Container', special=True) @@ -1224,6 +1174,56 @@ def __round__(self, ndigits: int = 0) -> T_co: Generator = _GenericAlias(collections.abc.Generator, (T_co, T_contra, V_co), name='Generator', special=True) AsyncGenerator = _GenericAlias(collections.abc.AsyncGenerator, (T_co, T_contra), name='AsyncGenerator', special=True) + + +class SupportsInt(_Protocol): + __slots__ = () + + @abstractmethod + def __int__(self) -> int: + pass + + +class SupportsFloat(_Protocol): + __slots__ = () + + @abstractmethod + def __float__(self) -> float: + pass + + +class SupportsComplex(_Protocol): + __slots__ = () + + @abstractmethod + def __complex__(self) -> complex: + pass + + +class SupportsBytes(_Protocol): + __slots__ = () + + @abstractmethod + def __bytes__(self) -> bytes: + pass + + +class SupportsAbs(_Protocol[T_co]): + __slots__ = () + + @abstractmethod + def __abs__(self) -> T_co: + pass + + +class SupportsRound(_Protocol[T_co]): + __slots__ = () + + @abstractmethod + def __round__(self, ndigits: int = 0) -> T_co: + pass + + # Internal type variable used for Type[]. CT_co = TypeVar('CT_co', covariant=True, bound=type) From 57855bcf4ff4f737cddafa9432c454df6ddc323f Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 14 Nov 2017 14:18:45 +0100 Subject: [PATCH 51/82] Wrap lines longer than 90 --- Lib/typing.py | 117 ++++++++++++++++++++++++++++++++------------------ 1 file changed, 76 insertions(+), 41 deletions(-) diff --git a/Lib/typing.py b/Lib/typing.py index c563dac2952c2e..debfc9b9e42869 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -124,7 +124,8 @@ def _type_check(arg, msg): return ForwardRef(arg) if ( # Bare Union etc. are not valid as type arguments - _GenericAlias and isinstance(arg, _GenericAlias) and arg.__origin__ in (Generic, _Protocol, ClassVar) or + _GenericAlias and isinstance(arg, _GenericAlias) and + arg.__origin__ in (Generic, _Protocol, ClassVar) or arg in (Generic, _Protocol, ClassVar, Union, NoReturn, Optional) ): raise TypeError("Plain %s is not valid as type argument" % arg) @@ -204,7 +205,8 @@ def _remove_dups_flatten(parameters): for t1 in params: if not isinstance(t1, type): continue - if any((isinstance(t2, type) or isinstance(t2, _GenericAlias) and t2._special) and issubclass(t1, t2) + if any((isinstance(t2, type) or + isinstance(t2, _GenericAlias) and t2._special) and issubclass(t1, t2) for t2 in all_params - {t1}): all_params.remove(t1) return tuple(t for t in params if t in all_params) @@ -572,7 +574,8 @@ def _is_dunder(attr): class _GenericAlias(_Final, _root=True): - def __init__(self, origin, params, *, name=None, subcls=True, inst=True, special=False): + def __init__(self, origin, params, *, + name=None, subcls=True, inst=True, special=False): self._name = name self._subcls = subcls self._inst = inst @@ -609,7 +612,8 @@ def __getitem__(self, params): params = (Ellipsis, result) else: if not isinstance(args, list): - raise TypeError(f"Callable[args, result]: args must be a list. Got {args}") + raise TypeError(f"Callable[args, result]: args must be a list." + f" Got {args}") params = (tuple(args), result) return self.__getitem_inner__(params) @@ -620,16 +624,19 @@ def __getitem_inner__(self, params): raise TypeError("Cannot subscript already-subscripted {self}") if self.__origin__ is tuple and self._special: if params == (): - return _GenericAlias(tuple, (_TypingEmpty,), name=self._name, inst=self._inst, subcls=self._subcls) + return _GenericAlias(tuple, (_TypingEmpty,), name=self._name, + inst=self._inst, subcls=self._subcls) if not isinstance(params, tuple): params = (params,) if len(params) == 2 and params[1] is ...: msg = "Tuple[t, ...]: t must be a type." p = _type_check(params[0], msg) - return _GenericAlias(tuple, (p, _TypingEllipsis), name=self._name, inst=self._inst, subcls=self._subcls) + return _GenericAlias(tuple, (p, _TypingEllipsis), name=self._name, + inst=self._inst, subcls=self._subcls) msg = "Tuple[t0, t1, ...]: each t must be a type." params = tuple(_type_check(p, msg) for p in params) - return _GenericAlias(tuple, params, name=self._name, inst=self._inst, subcls=self._subcls) + return _GenericAlias(tuple, params, name=self._name, + inst=self._inst, subcls=self._subcls) if self.__origin__ is collections.abc.Callable and self._special: args, result = params msg = "Callable[args, result]: result must be a type." @@ -640,7 +647,8 @@ def __getitem_inner__(self, params): msg = "Callable[[arg, ...], result]: each arg must be a type." args = tuple(_type_check(arg, msg) for arg in args) params = args + (result,) - return _GenericAlias(self.__origin__, params, name=self._name, inst=self._inst, subcls=self._subcls) + return _GenericAlias(self.__origin__, params, name=self._name, + inst=self._inst, subcls=self._subcls) if not isinstance(params, tuple): params = (params,) msg = "Parameters to generic types must be types." @@ -710,7 +718,8 @@ def __mro_entries__(self, bases): return (self.__origin__,) def __getattr__(self, attr): - if '__origin__' in self.__dict__ and not _is_dunder(attr): # We are carefull for copy and pickle + # We are carefull for copy and pickle. + if '__origin__' in self.__dict__ and not _is_dunder(attr): return getattr(self.__origin__, attr) raise AttributeError(attr) @@ -728,7 +737,12 @@ def __subclasscheck__(self, cls): return issubclass(cls, self.__origin__) if cls._special: return issubclass(cls.__origin__, self.__origin__) - raise TypeError("Subscripted generics cannot be used with class and instance checks") + raise TypeError("Subscripted generics cannot be used with" + " class and instance checks") + + +class _VariadicGenericAlias(_GenericAlias, _root=True): + pass class Generic: @@ -777,10 +791,8 @@ def __class_getitem__(cls, params): if len(set(params)) != len(params): raise TypeError( "Parameters to Generic[...] must all be unique") - elif cls in (Tuple, Callable): - pass elif cls is _Protocol: - # _Protocol is internal, don't check anything. + # _Protocol is internal at the moment, just skip the check pass else: # Subscripting a regular Generic subclass. @@ -1116,18 +1128,28 @@ def __class_getitem__(cls, params): # Various ABCs mimicking those in collections.abc. -Hashable = _GenericAlias(collections.abc.Hashable, (), name='Hashable', special=True) # Not generic. -Awaitable = _GenericAlias(collections.abc.Awaitable, T_co, name='Awaitable', special=True) -Coroutine = _GenericAlias(collections.abc.Coroutine, (T_co, T_contra, V_co), name='Coroutine', special=True) -AsyncIterable = _GenericAlias(collections.abc.AsyncIterable, T_co, name='AsyncIterable', special=True) -AsyncIterator = _GenericAlias(collections.abc.AsyncIterator, T_co, name='AsyncIterator', special=True) +Hashable = _GenericAlias(collections.abc.Hashable, (), + name='Hashable', special=True) # Not generic. +Awaitable = _GenericAlias(collections.abc.Awaitable, T_co, + name='Awaitable', special=True) +Coroutine = _GenericAlias(collections.abc.Coroutine, (T_co, T_contra, V_co), + name='Coroutine', special=True) +AsyncIterable = _GenericAlias(collections.abc.AsyncIterable, T_co, + name='AsyncIterable', special=True) +AsyncIterator = _GenericAlias(collections.abc.AsyncIterator, T_co, + name='AsyncIterator', special=True) Iterable = _GenericAlias(collections.abc.Iterable, T_co, name='Iterable', special=True) Iterator = _GenericAlias(collections.abc.Iterator, T_co, name='Iterator', special=True) -Reversible = _GenericAlias(collections.abc.Reversible, T_co, name='Reversible', special=True) -Sized = _GenericAlias(collections.abc.Sized, (), name='Sized', special=True) # Not generic. -Container = _GenericAlias(collections.abc.Container, T_co, name='Container', special=True) -Collection = _GenericAlias(collections.abc.Collection, T_co, name='Collection', special=True) -Callable = _GenericAlias(collections.abc.Callable, (), name='Callable', special=True) +Reversible = _GenericAlias(collections.abc.Reversible, T_co, + name='Reversible', special=True) +Sized = _GenericAlias(collections.abc.Sized, (), + name='Sized', special=True) # Not generic. +Container = _GenericAlias(collections.abc.Container, T_co, + name='Container', special=True) +Collection = _GenericAlias(collections.abc.Collection, T_co, + name='Collection', special=True) +Callable = _VariadicGenericAlias(collections.abc.Callable, (), + name='Callable', special=True) Callable.__doc__ = \ """Callable type; Callable[[int], str] is a function of (int) -> str. @@ -1139,15 +1161,19 @@ def __class_getitem__(cls, params): such function types are rarely used as callback types. """ AbstractSet = _GenericAlias(collections.abc.Set, T_co, name='AbstractSet', special=True) -MutableSet = _GenericAlias(collections.abc.MutableSet, T, name='MutableSet', special=True) +MutableSet = _GenericAlias(collections.abc.MutableSet, T, + name='MutableSet', special=True) # NOTE: Mapping is only covariant in the value type. -Mapping = _GenericAlias(collections.abc.Mapping, (KT, VT_co), name='Mapping', special=True) -MutableMapping = _GenericAlias(collections.abc.MutableMapping, (KT, VT), name='MutableMapping', special=True) +Mapping = _GenericAlias(collections.abc.Mapping, (KT, VT_co), + name='Mapping', special=True) +MutableMapping = _GenericAlias(collections.abc.MutableMapping, (KT, VT), + name='MutableMapping', special=True) Sequence = _GenericAlias(collections.abc.Sequence, T_co, name='Sequence', special=True) -MutableSequence = _GenericAlias(collections.abc.MutableSequence, T, name='MutableSequence', special=True) -# Not generic -ByteString = _GenericAlias(collections.abc.ByteString, (), name='ByteString', special=True) -Tuple = _GenericAlias(tuple, (), name='Tuple', inst=False, special=True) +MutableSequence = _GenericAlias(collections.abc.MutableSequence, T, + name='MutableSequence', special=True) +ByteString = _GenericAlias(collections.abc.ByteString, (), + name='ByteString', special=True) # Not generic +Tuple = _VariadicGenericAlias(tuple, (), name='Tuple', inst=False, special=True) Tuple.__doc__ = \ """Tuple type; Tuple[X, Y] is the cross-product type of X and Y. @@ -1161,19 +1187,26 @@ def __class_getitem__(cls, params): Deque = _GenericAlias(collections.deque, T, name='Deque', special=True) Set = _GenericAlias(set, T, name='Set', inst=False, special=True) FrozenSet = _GenericAlias(frozenset, T_co, name='FrozenSet', inst=False, special=True) -MappingView = _GenericAlias(collections.abc.MappingView, T_co, name='MappingView', special=True) +MappingView = _GenericAlias(collections.abc.MappingView, T_co, + name='MappingView', special=True) KeysView = _GenericAlias(collections.abc.KeysView, KT, name='KeysView', special=True) -ItemsView = _GenericAlias(collections.abc.ItemsView, (KT, VT_co), name='ItemsView', special=True) -ValuesView = _GenericAlias(collections.abc.ValuesView, VT_co, name='ValuesView', special=True) -ContextManager = _GenericAlias(contextlib.AbstractContextManager, T_co, name='ContextManager', special=True) -#AsyncContextManager = _GenericAlias(contextlib.AbstractAsyncContextManager, T_co, name='AsyncContextManager') +ItemsView = _GenericAlias(collections.abc.ItemsView, (KT, VT_co), + name='ItemsView', special=True) +ValuesView = _GenericAlias(collections.abc.ValuesView, VT_co, + name='ValuesView', special=True) +ContextManager = _GenericAlias(contextlib.AbstractContextManager, T_co, + name='ContextManager', special=True) +#AsyncContextManager = _GenericAlias(contextlib.AbstractAsyncContextManager, T_co, +# name='AsyncContextManager') Dict = _GenericAlias(dict, (KT, VT), name='Dict', inst=False, special=True) -DefaultDict = _GenericAlias(collections.defaultdict, (KT, VT), name='DefaultDict', special=True) +DefaultDict = _GenericAlias(collections.defaultdict, (KT, VT), + name='DefaultDict', special=True) Counter = _GenericAlias(collections.Counter, T, name='Counter', special=True) ChainMap = _GenericAlias(collections.ChainMap, (KT, VT), name='ChainMap', special=True) -Generator = _GenericAlias(collections.abc.Generator, (T_co, T_contra, V_co), name='Generator', special=True) -AsyncGenerator = _GenericAlias(collections.abc.AsyncGenerator, (T_co, T_contra), name='AsyncGenerator', special=True) - +Generator = _GenericAlias(collections.abc.Generator, (T_co, T_contra, V_co), + name='Generator', special=True) +AsyncGenerator = _GenericAlias(collections.abc.AsyncGenerator, (T_co, T_contra), + name='AsyncGenerator', special=True) class SupportsInt(_Protocol): @@ -1540,8 +1573,10 @@ class io: io.__name__ = __name__ + '.io' sys.modules[io.__name__] = io -Pattern = _GenericAlias(type(stdlib_re.compile('')), AnyStr, name='Pattern', subcls=False, special=True) -Match = _GenericAlias(type(stdlib_re.match('', '')), AnyStr, name='Match', subcls=False, special=True) +Pattern = _GenericAlias(type(stdlib_re.compile('')), AnyStr, + name='Pattern', subcls=False, special=True) +Match = _GenericAlias(type(stdlib_re.match('', '')), AnyStr, + name='Match', subcls=False, special=True) class re: """Wrapper namespace for re type aliases.""" From 05553266085a1952cba515051c20196a9f8243cf Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 14 Nov 2017 14:45:26 +0100 Subject: [PATCH 52/82] Factor out variadic aliases; use slots for special forms --- Lib/typing.py | 109 +++++++++++++++++++++++++++++--------------------- 1 file changed, 63 insertions(+), 46 deletions(-) diff --git a/Lib/typing.py b/Lib/typing.py index debfc9b9e42869..3b1e8bb9cf2e57 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -247,13 +247,26 @@ def inner(*args, **kwds): class _Final: """Mixin to prohibit subclassing""" + __slots__ = ('__weakref__',) + def __init_subclass__(self, *args, **kwds): if '_root' not in kwds: raise TypeError("Cannot subclass special typing classes") class _SpecialForm(_Final, _root=True): - """Internal indicator of special typing constructs.""" + """Internal indicator of special typing constructs. + See _doc instance attribute for specific docs. + """ + + __slots__ = ('_name', '_doc') + + def __getstate__(self): + return {'name': self._name, 'doc': self._doc} + + def __setstate__(self, state): + self._name = state['name'] + self._doc = state['doc'] def __new__(cls, *args, **kwds): """Constructor. @@ -587,6 +600,7 @@ def __init__(self, origin, params, *, () if a is _TypingEmpty else a for a in params) self.__parameters__ = _type_vars(params) + self.__slots__ = None # This is not documented. if not name: self.__module__ = origin.__module__ @@ -601,54 +615,11 @@ def _eval_type(self, globalns, localns): return _GenericAlias(self.__origin__, ev_args, name=self._name, subcls=self._subcls, inst=self._inst, special=self._special) - def __getitem__(self, params): - if self._name != 'Callable' or not self._special: - return self.__getitem_inner__(params) - if not isinstance(params, tuple) or len(params) != 2: - raise TypeError("Callable must be used as " - "Callable[[arg, ...], result].") - args, result = params - if args is Ellipsis: - params = (Ellipsis, result) - else: - if not isinstance(args, list): - raise TypeError(f"Callable[args, result]: args must be a list." - f" Got {args}") - params = (tuple(args), result) - return self.__getitem_inner__(params) - @_tp_cache - def __getitem_inner__(self, params): + def __getitem__(self, params): if self.__origin__ in (Generic, _Protocol): # Can't subscript Generic[...] or _Protocol[...]. raise TypeError("Cannot subscript already-subscripted {self}") - if self.__origin__ is tuple and self._special: - if params == (): - return _GenericAlias(tuple, (_TypingEmpty,), name=self._name, - inst=self._inst, subcls=self._subcls) - if not isinstance(params, tuple): - params = (params,) - if len(params) == 2 and params[1] is ...: - msg = "Tuple[t, ...]: t must be a type." - p = _type_check(params[0], msg) - return _GenericAlias(tuple, (p, _TypingEllipsis), name=self._name, - inst=self._inst, subcls=self._subcls) - msg = "Tuple[t0, t1, ...]: each t must be a type." - params = tuple(_type_check(p, msg) for p in params) - return _GenericAlias(tuple, params, name=self._name, - inst=self._inst, subcls=self._subcls) - if self.__origin__ is collections.abc.Callable and self._special: - args, result = params - msg = "Callable[args, result]: result must be a type." - result = _type_check(result, msg) - if args is Ellipsis: - return _GenericAlias(self.__origin__, (_TypingEllipsis, result), - name=self._name, inst=self._inst, subcls=self._subcls) - msg = "Callable[[arg, ...], result]: each arg must be a type." - args = tuple(_type_check(arg, msg) for arg in args) - params = args + (result,) - return _GenericAlias(self.__origin__, params, name=self._name, - inst=self._inst, subcls=self._subcls) if not isinstance(params, tuple): params = (params,) msg = "Parameters to generic types must be types." @@ -742,7 +713,53 @@ def __subclasscheck__(self, cls): class _VariadicGenericAlias(_GenericAlias, _root=True): - pass + + def __getitem__(self, params): + if self._name != 'Callable' or not self._special: + return self.__getitem_inner__(params) + if not isinstance(params, tuple) or len(params) != 2: + raise TypeError("Callable must be used as " + "Callable[[arg, ...], result].") + args, result = params + if args is Ellipsis: + params = (Ellipsis, result) + else: + if not isinstance(args, list): + raise TypeError(f"Callable[args, result]: args must be a list." + f" Got {args}") + params = (tuple(args), result) + return self.__getitem_inner__(params) + + @_tp_cache + def __getitem_inner__(self, params): + if self.__origin__ is tuple and self._special: + if params == (): + return _GenericAlias(tuple, (_TypingEmpty,), name=self._name, + inst=self._inst, subcls=self._subcls) + if not isinstance(params, tuple): + params = (params,) + if len(params) == 2 and params[1] is ...: + msg = "Tuple[t, ...]: t must be a type." + p = _type_check(params[0], msg) + return _GenericAlias(tuple, (p, _TypingEllipsis), name=self._name, + inst=self._inst, subcls=self._subcls) + msg = "Tuple[t0, t1, ...]: each t must be a type." + params = tuple(_type_check(p, msg) for p in params) + return _GenericAlias(tuple, params, name=self._name, + inst=self._inst, subcls=self._subcls) + if self.__origin__ is collections.abc.Callable and self._special: + args, result = params + msg = "Callable[args, result]: result must be a type." + result = _type_check(result, msg) + if args is Ellipsis: + return _GenericAlias(self.__origin__, (_TypingEllipsis, result), + name=self._name, inst=self._inst, subcls=self._subcls) + msg = "Callable[[arg, ...], result]: each arg must be a type." + args = tuple(_type_check(arg, msg) for arg in args) + params = args + (result,) + return _GenericAlias(self.__origin__, params, name=self._name, + inst=self._inst, subcls=self._subcls) + return super().__getitem__(params) class Generic: From 23e63e0c6d10da82046b8a7a7e27aa2cb444893b Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 14 Nov 2017 14:47:50 +0100 Subject: [PATCH 53/82] Factor out variadic aliases; use slots for special forms --- Lib/typing.py | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/Lib/typing.py b/Lib/typing.py index 3b1e8bb9cf2e57..07d452de226171 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -628,22 +628,15 @@ def __getitem__(self, params): return _subs_tvars(self, self.__parameters__, params) def __repr__(self): - if (self._name != 'Callable' or - len(self.__args__) == 2 and self.__args__[0] is Ellipsis): - if self._name: - name = 'typing.' + self._name - else: - name = _type_repr(self.__origin__) - if not self._special: - args = f'[{", ".join([_type_repr(a) for a in self.__args__])}]' - else: - args = '' - return (f'{name}{args}') - if self._special: - return 'typing.Callable' - return (f'typing.Callable' - f'[[{", ".join([_type_repr(a) for a in self.__args__[:-1]])}], ' - f'{_type_repr(self.__args__[-1])}]') + if self._name: + name = 'typing.' + self._name + else: + name = _type_repr(self.__origin__) + if not self._special: + args = f'[{", ".join([_type_repr(a) for a in self.__args__])}]' + else: + args = '' + return (f'{name}{args}') def __eq__(self, other): if not isinstance(other, _GenericAlias): @@ -714,6 +707,16 @@ def __subclasscheck__(self, cls): class _VariadicGenericAlias(_GenericAlias, _root=True): + def __repr__(self): + if (self._name != 'Callable' or + len(self.__args__) == 2 and self.__args__[0] is Ellipsis): + return super().__repr__() + if self._special: + return 'typing.Callable' + return (f'typing.Callable' + f'[[{", ".join([_type_repr(a) for a in self.__args__[:-1]])}], ' + f'{_type_repr(self.__args__[-1])}]') + def __getitem__(self, params): if self._name != 'Callable' or not self._special: return self.__getitem_inner__(params) From fb0bde780a2b34cf4b0294952755bb468ebacf31 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 14 Nov 2017 14:51:32 +0100 Subject: [PATCH 54/82] Revert the repr part --- Lib/typing.py | 35 ++++++++++++++++------------------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/Lib/typing.py b/Lib/typing.py index 07d452de226171..3b1e8bb9cf2e57 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -628,15 +628,22 @@ def __getitem__(self, params): return _subs_tvars(self, self.__parameters__, params) def __repr__(self): - if self._name: - name = 'typing.' + self._name - else: - name = _type_repr(self.__origin__) - if not self._special: - args = f'[{", ".join([_type_repr(a) for a in self.__args__])}]' - else: - args = '' - return (f'{name}{args}') + if (self._name != 'Callable' or + len(self.__args__) == 2 and self.__args__[0] is Ellipsis): + if self._name: + name = 'typing.' + self._name + else: + name = _type_repr(self.__origin__) + if not self._special: + args = f'[{", ".join([_type_repr(a) for a in self.__args__])}]' + else: + args = '' + return (f'{name}{args}') + if self._special: + return 'typing.Callable' + return (f'typing.Callable' + f'[[{", ".join([_type_repr(a) for a in self.__args__[:-1]])}], ' + f'{_type_repr(self.__args__[-1])}]') def __eq__(self, other): if not isinstance(other, _GenericAlias): @@ -707,16 +714,6 @@ def __subclasscheck__(self, cls): class _VariadicGenericAlias(_GenericAlias, _root=True): - def __repr__(self): - if (self._name != 'Callable' or - len(self.__args__) == 2 and self.__args__[0] is Ellipsis): - return super().__repr__() - if self._special: - return 'typing.Callable' - return (f'typing.Callable' - f'[[{", ".join([_type_repr(a) for a in self.__args__[:-1]])}], ' - f'{_type_repr(self.__args__[-1])}]') - def __getitem__(self, params): if self._name != 'Callable' or not self._special: return self.__getitem_inner__(params) From 2f07026d11b5dad69e2b13cfebac85f3eeba8c22 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 14 Nov 2017 15:56:38 +0100 Subject: [PATCH 55/82] Docstring formatting --- Lib/typing.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Lib/typing.py b/Lib/typing.py index 3b1e8bb9cf2e57..7f21b6ca06f0c8 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -111,8 +111,7 @@ def _type_check(arg, msg): As a special case, accept None and return type(None) instead. Also, _TypeAlias instances (e.g. Match, Pattern) are acceptable. - - The msg argument is a human-readable error message, e.g. + The msg argument is a human-readable error message, e.g:: "Union[arg, ...]: arg should be a type." From d4f825886da2a786c6aa47b1c3d5320458d37c8c Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 14 Nov 2017 16:36:18 +0100 Subject: [PATCH 56/82] Simplify type vars collection --- Lib/typing.py | 57 +++++++++++++++++++++------------------------------ 1 file changed, 23 insertions(+), 34 deletions(-) diff --git a/Lib/typing.py b/Lib/typing.py index 7f21b6ca06f0c8..cbbfd98a6161c7 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -85,15 +85,13 @@ # legitimate imports of those modules. -def _get_type_vars(types, tvars): - for t in types: - if isinstance(t, (_GenericAlias, TypeVar)): - t._get_type_vars(tvars) - - -def _type_vars(types): +def _collect_type_vars(types): tvars = [] - _get_type_vars(types, tvars) + for t in types: + if isinstance(t, TypeVar) and t not in tvars: + tvars.append(t) + if isinstance(t, _GenericAlias) and not t._special: + tvars.extend([t for t in t.__parameters__ if t not in tvars]) return tuple(tvars) @@ -167,8 +165,7 @@ def _subs_tvars(tp, tvars, subs): new_args[a] = _subs_tvars(arg, tvars, subs) if tp.__origin__ is Union: return Union[tuple(new_args)] - return _GenericAlias(tp.__origin__, tuple(new_args), - name=tp._name, subcls=tp._subcls, inst=tp._inst) + return tp.copy_with(tuple(new_args)) def _remove_dups_flatten(parameters): @@ -541,10 +538,6 @@ def __init__(self, name, *constraints, bound=None, else: self.__bound__ = None - def _get_type_vars(self, tvars): - if self not in tvars: - tvars.append(self) - def __repr__(self): if self.__covariant__: prefix = '+' @@ -579,6 +572,7 @@ def __repr__(self): # e.g., Dict[T, int].__args__ == (T, int). _FLAGS = ('_name', '_subcls', '_inst', '_special') +_sentinel = object() def _is_dunder(attr): return attr.startswith('__') and attr.endswith('__') @@ -598,27 +592,22 @@ def __init__(self, origin, params, *, self.__args__ = tuple(... if a is _TypingEllipsis else () if a is _TypingEmpty else a for a in params) - self.__parameters__ = _type_vars(params) + self.__parameters__ = _collect_type_vars(params) self.__slots__ = None # This is not documented. if not name: self.__module__ = origin.__module__ - def _get_type_vars(self, tvars): - if not self._special: - _get_type_vars(self.__parameters__, tvars) - def _eval_type(self, globalns, localns): ev_args = tuple(_eval_type(a, globalns, localns) for a in self.__args__) if ev_args == self.__args__: return self - return _GenericAlias(self.__origin__, ev_args, name=self._name, - subcls=self._subcls, inst=self._inst, special=self._special) + return self.copy_with(ev_args) @_tp_cache def __getitem__(self, params): if self.__origin__ in (Generic, _Protocol): # Can't subscript Generic[...] or _Protocol[...]. - raise TypeError("Cannot subscript already-subscripted {self}") + raise TypeError(f"Cannot subscript already-subscripted {self}") if not isinstance(params, tuple): params = (params,) msg = "Parameters to generic types must be types." @@ -626,6 +615,11 @@ def __getitem__(self, params): _check_generic(self, params) return _subs_tvars(self, self.__parameters__, params) + def copy_with(self, params): + # We don't copy _special. + return _GenericAlias(self.__origin__, params, name=self._name, + subcls=self._subcls, inst=self._inst) + def __repr__(self): if (self._name != 'Callable' or len(self.__args__) == 2 and self.__args__[0] is Ellipsis): @@ -696,7 +690,7 @@ def __getattr__(self, attr): def __setattr__(self, attr, val): if not _is_dunder(attr) and attr not in _FLAGS: setattr(self.__origin__, attr, val) - self.__dict__[attr] = val + super().__setattr__(attr, val) def __instancecheck__(self, obj): return self.__subclasscheck__(type(obj)) @@ -733,31 +727,26 @@ def __getitem__(self, params): def __getitem_inner__(self, params): if self.__origin__ is tuple and self._special: if params == (): - return _GenericAlias(tuple, (_TypingEmpty,), name=self._name, - inst=self._inst, subcls=self._subcls) + return self.copy_with((_TypingEmpty,)) if not isinstance(params, tuple): params = (params,) if len(params) == 2 and params[1] is ...: msg = "Tuple[t, ...]: t must be a type." p = _type_check(params[0], msg) - return _GenericAlias(tuple, (p, _TypingEllipsis), name=self._name, - inst=self._inst, subcls=self._subcls) + return self.copy_with((p, _TypingEllipsis)) msg = "Tuple[t0, t1, ...]: each t must be a type." params = tuple(_type_check(p, msg) for p in params) - return _GenericAlias(tuple, params, name=self._name, - inst=self._inst, subcls=self._subcls) + return self.copy_with(params) if self.__origin__ is collections.abc.Callable and self._special: args, result = params msg = "Callable[args, result]: result must be a type." result = _type_check(result, msg) if args is Ellipsis: - return _GenericAlias(self.__origin__, (_TypingEllipsis, result), - name=self._name, inst=self._inst, subcls=self._subcls) + return self.copy_with((_TypingEllipsis, result)) msg = "Callable[[arg, ...], result]: each arg must be a type." args = tuple(_type_check(arg, msg) for arg in args) params = args + (result,) - return _GenericAlias(self.__origin__, params, name=self._name, - inst=self._inst, subcls=self._subcls) + return self.copy_with(params) return super().__getitem__(params) @@ -822,7 +811,7 @@ def __init_subclass__(cls, *args, **kwargs): hasattr(cls, '__orig_bases__') and Generic in cls.__orig_bases__): raise TypeError("Cannot inherit from plain Generic") if '__orig_bases__' in cls.__dict__: - tvars = _type_vars(cls.__orig_bases__) + tvars = _collect_type_vars(cls.__orig_bases__) # Look for Generic[T1, ..., Tn]. # If found, tvars must be a subset of it. # If not found, tvars is it. From c6f51fe9175171bf5c933d0607ac487022884db5 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 14 Nov 2017 16:38:58 +0100 Subject: [PATCH 57/82] Simplify type vars collection --- Lib/typing.py | 63 ++++++++++++++++++++++++--------------------------- 1 file changed, 30 insertions(+), 33 deletions(-) diff --git a/Lib/typing.py b/Lib/typing.py index cbbfd98a6161c7..58bcd58eff79af 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -85,25 +85,6 @@ # legitimate imports of those modules. -def _collect_type_vars(types): - tvars = [] - for t in types: - if isinstance(t, TypeVar) and t not in tvars: - tvars.append(t) - if isinstance(t, _GenericAlias) and not t._special: - tvars.extend([t for t in t.__parameters__ if t not in tvars]) - return tuple(tvars) - - -def _eval_type(t, globalns, localns): - if isinstance(t, (_GenericAlias, ForwardRef)): - return t._eval_type(globalns, localns) - return t - -_GenericAlias = None -Generic = object() -_Protocol = object() - def _type_check(arg, msg): """Check that the argument is a type, and return it (internal helper). @@ -208,6 +189,22 @@ def _remove_dups_flatten(parameters): return tuple(t for t in params if t in all_params) +def _collect_type_vars(types): + tvars = [] + for t in types: + if isinstance(t, TypeVar) and t not in tvars: + tvars.append(t) + if isinstance(t, _GenericAlias) and not t._special: + tvars.extend([t for t in t.__parameters__ if t not in tvars]) + return tuple(tvars) + + +def _eval_type(t, globalns, localns): + if isinstance(t, (_GenericAlias, ForwardRef)): + return t._eval_type(globalns, localns) + return t + + def _check_generic(cls, parameters): # Check correct count for parameters of a generic cls (internal helper). if not cls.__parameters__: @@ -548,20 +545,6 @@ def __repr__(self): return prefix + self.__name__ -# Some unconstrained type variables. These are used by the container types. -# (These are not for export.) -T = TypeVar('T') # Any type. -KT = TypeVar('KT') # Key type. -VT = TypeVar('VT') # Value type. -T_co = TypeVar('T_co', covariant=True) # Any type covariant containers. -V_co = TypeVar('V_co', covariant=True) # Any type covariant containers. -VT_co = TypeVar('VT_co', covariant=True) # Value type covariant containers. -T_contra = TypeVar('T_contra', contravariant=True) # Ditto contravariant. - -# A useful type variable with constraints. This represents string types. -# (This one *is* for export!) -AnyStr = TypeVar('AnyStr', bytes, str) - # Special typing constructs Union, Optional, Generic, Callable and Tuple # use three special attributes for internal bookkeeping of generic types: # * __parameters__ is a tuple of unique free type parameters of a generic @@ -1132,6 +1115,20 @@ def __class_getitem__(cls, params): return Generic.__class_getitem__(cls, params) +# Some unconstrained type variables. These are used by the container types. +# (These are not for export.) +T = TypeVar('T') # Any type. +KT = TypeVar('KT') # Key type. +VT = TypeVar('VT') # Value type. +T_co = TypeVar('T_co', covariant=True) # Any type covariant containers. +V_co = TypeVar('V_co', covariant=True) # Any type covariant containers. +VT_co = TypeVar('VT_co', covariant=True) # Value type covariant containers. +T_contra = TypeVar('T_contra', contravariant=True) # Ditto contravariant. + +# A useful type variable with constraints. This represents string types. +# (This one *is* for export!) +AnyStr = TypeVar('AnyStr', bytes, str) + # Various ABCs mimicking those in collections.abc. Hashable = _GenericAlias(collections.abc.Hashable, (), name='Hashable', special=True) # Not generic. From 8afdee28ca9248a700adba53d19ce969676a5c37 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 14 Nov 2017 17:06:17 +0100 Subject: [PATCH 58/82] Simplify type evaluation --- Lib/typing.py | 112 ++++++++++++++++++++++++++++---------------------- 1 file changed, 62 insertions(+), 50 deletions(-) diff --git a/Lib/typing.py b/Lib/typing.py index 58bcd58eff79af..07c716d91bb9ef 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -133,7 +133,25 @@ def _type_repr(obj): return repr(obj) +def _collect_type_vars(types): + """Collect all type variable contained in types in order of + first appearance (lexicographic order). For example:: + + _collect_type_vars((T, List[S, T])) == (T, S) + """ + tvars = [] + for t in types: + if isinstance(t, TypeVar) and t not in tvars: + tvars.append(t) + if isinstance(t, _GenericAlias) and not t._special: + tvars.extend([t for t in t.__parameters__ if t not in tvars]) + return tuple(tvars) + + def _subs_tvars(tp, tvars, subs): + """Substitute type variables 'tvars' with substitutions 'subs'. + These two must have same length. + """ if not isinstance(tp, _GenericAlias): return tp new_args = list(tp.__args__) @@ -149,11 +167,23 @@ def _subs_tvars(tp, tvars, subs): return tp.copy_with(tuple(new_args)) +def _check_generic(cls, parameters): + """Check correct count for parameters of a generic cls (internal helper). + This gives a nice error message in case of count mismatch. + """ + if not cls.__parameters__: + raise TypeError("%s is not a generic class" % repr(cls)) + alen = len(parameters) + elen = len(cls.__parameters__) + if alen != elen: + raise TypeError("Too %s parameters for %s; actual %s, expected %s" % + ("many" if alen > elen else "few", repr(cls), alen, elen)) + + def _remove_dups_flatten(parameters): """An internal helper for Union creation and substitution: flatten Union's among parameters, then remove duplicates and strict subclasses. """ - # Flatten out Union[Union[...], ...]. params = [] for p in parameters: @@ -189,33 +219,6 @@ def _remove_dups_flatten(parameters): return tuple(t for t in params if t in all_params) -def _collect_type_vars(types): - tvars = [] - for t in types: - if isinstance(t, TypeVar) and t not in tvars: - tvars.append(t) - if isinstance(t, _GenericAlias) and not t._special: - tvars.extend([t for t in t.__parameters__ if t not in tvars]) - return tuple(tvars) - - -def _eval_type(t, globalns, localns): - if isinstance(t, (_GenericAlias, ForwardRef)): - return t._eval_type(globalns, localns) - return t - - -def _check_generic(cls, parameters): - # Check correct count for parameters of a generic cls (internal helper). - if not cls.__parameters__: - raise TypeError("%s is not a generic class" % repr(cls)) - alen = len(parameters) - elen = len(cls.__parameters__) - if alen != elen: - raise TypeError("Too %s parameters for %s; actual %s, expected %s" % - ("many" if alen > elen else "few", repr(cls), alen, elen)) - - _cleanups = [] @@ -223,7 +226,6 @@ def _tp_cache(func): """Internal wrapper caching __getitem__ of generic types with a fallback to original function for non-hashable arguments. """ - cached = functools.lru_cache()(func) _cleanups.append(cached.cache_clear) @@ -237,6 +239,22 @@ def inner(*args, **kwds): return inner +def _eval_type(t, globalns, localns): + """Evaluate all forward reverences in the given type t. + For use of globalns and localns see the docstring for get_type_hints(). + """ + if isinstance(t, ForwardRef): + return t._evaluate(globalns, localns) + if isinstance(t, _GenericAlias): + ev_args = tuple(_eval_type(a, globalns, localns) for a in t.__args__) + if ev_args == t.__args__: + return t + res = t.copy_with(ev_args) + res._special = t._special + return res + return t + + class _Final: """Mixin to prohibit subclassing""" @@ -431,7 +449,7 @@ def __init__(self, arg): self.__forward_evaluated__ = False self.__forward_value__ = None - def _eval_type(self, globalns, localns): + def _evaluate(self, globalns, localns): if not self.__forward_evaluated__ or localns is not globalns: if globalns is None and localns is None: globalns = localns = {} @@ -503,20 +521,6 @@ def longest(x: A, y: A) -> A: __slots__ = ('__name__', '__bound__', '__constraints__', '__covariant__', '__contravariant__') - def __getstate__(self): - return {'name': self.__name__, - 'bound': self.__bound__, - 'constraints': self.__constraints__, - 'co': self.__covariant__, - 'contra': self.__contravariant__} - - def __setstate__(self, state): - self.__name__ = state['name'] - self.__bound__ = state['bound'] - self.__constraints__ = state['constraints'] - self.__covariant__ = state['co'] - self.__contravariant__ = state['contra'] - def __init__(self, name, *constraints, bound=None, covariant=False, contravariant=False): self.__name__ = name @@ -535,6 +539,20 @@ def __init__(self, name, *constraints, bound=None, else: self.__bound__ = None + def __getstate__(self): + return {'name': self.__name__, + 'bound': self.__bound__, + 'constraints': self.__constraints__, + 'co': self.__covariant__, + 'contra': self.__contravariant__} + + def __setstate__(self, state): + self.__name__ = state['name'] + self.__bound__ = state['bound'] + self.__constraints__ = state['constraints'] + self.__covariant__ = state['co'] + self.__contravariant__ = state['contra'] + def __repr__(self): if self.__covariant__: prefix = '+' @@ -580,12 +598,6 @@ def __init__(self, origin, params, *, if not name: self.__module__ = origin.__module__ - def _eval_type(self, globalns, localns): - ev_args = tuple(_eval_type(a, globalns, localns) for a in self.__args__) - if ev_args == self.__args__: - return self - return self.copy_with(ev_args) - @_tp_cache def __getitem__(self, params): if self.__origin__ in (Generic, _Protocol): From ab0f6780fa20c419bc7ba1fecf4300040ff2365d Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 14 Nov 2017 17:11:04 +0100 Subject: [PATCH 59/82] Abandon _subcls --- Lib/typing.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/Lib/typing.py b/Lib/typing.py index 07c716d91bb9ef..c656ab25c1dcc4 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -572,7 +572,6 @@ def __repr__(self): # * __args__ is a tuple of all arguments used in subscripting, # e.g., Dict[T, int].__args__ == (T, int). -_FLAGS = ('_name', '_subcls', '_inst', '_special') _sentinel = object() def _is_dunder(attr): @@ -582,9 +581,8 @@ def _is_dunder(attr): class _GenericAlias(_Final, _root=True): def __init__(self, origin, params, *, - name=None, subcls=True, inst=True, special=False): + name=None, inst=True, special=False): self._name = name - self._subcls = subcls self._inst = inst self._special = special if not isinstance(params, tuple): @@ -611,9 +609,8 @@ def __getitem__(self, params): return _subs_tvars(self, self.__parameters__, params) def copy_with(self, params): - # We don't copy _special. - return _GenericAlias(self.__origin__, params, name=self._name, - subcls=self._subcls, inst=self._inst) + # We don't copy self._special. + return _GenericAlias(self.__origin__, params, name=self._name, inst=self._inst) def __repr__(self): if (self._name != 'Callable' or @@ -683,7 +680,7 @@ def __getattr__(self, attr): raise AttributeError(attr) def __setattr__(self, attr, val): - if not _is_dunder(attr) and attr not in _FLAGS: + if not _is_dunder(attr) and attr not in ('_name', '_inst', '_special'): setattr(self.__origin__, attr, val) super().__setattr__(attr, val) @@ -1588,9 +1585,9 @@ class io: sys.modules[io.__name__] = io Pattern = _GenericAlias(type(stdlib_re.compile('')), AnyStr, - name='Pattern', subcls=False, special=True) + name='Pattern', special=True) Match = _GenericAlias(type(stdlib_re.match('', '')), AnyStr, - name='Match', subcls=False, special=True) + name='Match', special=True) class re: """Wrapper namespace for re type aliases.""" From 322e9c0e64355d2ca3e5b61300378b6bd831f5b8 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 14 Nov 2017 17:56:37 +0100 Subject: [PATCH 60/82] Last fixes; now all tests pass --- Lib/test/libregrtest/refleak.py | 5 --- Lib/test/test_typing.py | 4 +- Lib/typing.py | 75 +++++++++++++++++++-------------- 3 files changed, 44 insertions(+), 40 deletions(-) diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py index 18d5bd0511a76d..2ca9aa87644c0d 100644 --- a/Lib/test/libregrtest/refleak.py +++ b/Lib/test/libregrtest/refleak.py @@ -135,11 +135,6 @@ def dash_R_cleanup(fs, ps, pic, zdc, abcs): # Clear ABC registries, restoring previously saved ABC registries. abs_classes = [getattr(collections.abc, a) for a in collections.abc.__all__] abs_classes = filter(isabstract, abs_classes) - if 'typing' in sys.modules: - t = sys.modules['typing'] - # These classes require special treatment because they do not appear - # in direct subclasses of collections.abc classes - abs_classes = list(abs_classes) + [t.ChainMap, t.Counter, t.DefaultDict] for abc in abs_classes: for obj in abc.__subclasses__() + [abc]: obj._abc_registry = abcs.get(obj, WeakSet()).copy() diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index aeaa7db3dc0ec6..5adc2e41f1eefe 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -977,8 +977,6 @@ def test_fail_with_bare_generic(self): Tuple[Generic[T]] with self.assertRaises(TypeError): List[typing._Protocol] - with self.assertRaises(TypeError): - isinstance(1, Generic) def test_type_erasure_special(self): T = TypeVar('T') @@ -2453,7 +2451,7 @@ class A(typing.Match): pass self.assertEqual(str(ex.exception), - "Cannot subclass ") + "type 're.Match' is not an acceptable base type") class AllTests(BaseTestCase): diff --git a/Lib/typing.py b/Lib/typing.py index c656ab25c1dcc4..32fb7442ec876c 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -1,3 +1,20 @@ +""" +The typing module: + +* Imports +* Exports +* Internal helper functions +* _SpecialForm and its instances: Any, NoReturn, ClassVar, Union, Optional +* Two things that can be type arguments in addition to types: ForwardRef and TypeVar +* The central internal API: _GenericAlias, _VariadicGenericAlias +* The public counterpart of the API: Generic, Protocol (soon) +* Public functions: get_type_hints, overload, cast, no_type_check, + no_type_check_decorator +* Generic aliases for collections.abc ABCs and few additional protocols +* Special types: NewType, NamedTuple, TypedDict (soon) +* Wrapper re and io related types +""" + import abc from abc import abstractmethod, abstractproperty import collections @@ -1133,6 +1150,8 @@ def __class_getitem__(cls, params): V_co = TypeVar('V_co', covariant=True) # Any type covariant containers. VT_co = TypeVar('VT_co', covariant=True) # Value type covariant containers. T_contra = TypeVar('T_contra', contravariant=True) # Ditto contravariant. +# Internal type variable used for Type[]. +CT_co = TypeVar('CT_co', covariant=True, bound=type) # A useful type variable with constraints. This represents string types. # (This one *is* for export!) @@ -1218,6 +1237,30 @@ def __class_getitem__(cls, params): name='Generator', special=True) AsyncGenerator = _GenericAlias(collections.abc.AsyncGenerator, (T_co, T_contra), name='AsyncGenerator', special=True) +Type = _GenericAlias(type, CT_co, name='Type', inst=False, special=True) +Type.__doc__ = \ + """A special construct usable to annotate class objects. + + For example, suppose we have the following classes:: + + class User: ... # Abstract base for User classes + class BasicUser(User): ... + class ProUser(User): ... + class TeamUser(User): ... + + And a function that takes a class argument that's a subclass of + User and returns an instance of the corresponding class:: + + U = TypeVar('U', bound=User) + def new_user(user_class: Type[U]) -> U: + user = user_class() + # (Here we could write the user object to a database) + return user + + joe = new_user(BasicUser) + + At this point the type checker knows that joe has type BasicUser. + """ class SupportsInt(_Protocol): @@ -1268,38 +1311,6 @@ def __round__(self, ndigits: int = 0) -> T_co: pass -# Internal type variable used for Type[]. -CT_co = TypeVar('CT_co', covariant=True, bound=type) - - -# This is not a real generic class. Don't use outside annotations. -class Type(type, Generic[CT_co]): - """A special construct usable to annotate class objects. - - For example, suppose we have the following classes:: - - class User: ... # Abstract base for User classes - class BasicUser(User): ... - class ProUser(User): ... - class TeamUser(User): ... - - And a function that takes a class argument that's a subclass of - User and returns an instance of the corresponding class:: - - U = TypeVar('U', bound=User) - def new_user(user_class: Type[U]) -> U: - user = user_class() - # (Here we could write the user object to a database) - return user - - joe = new_user(BasicUser) - - At this point the type checker knows that joe has type BasicUser. - """ - - __slots__ = () - - def _make_nmtuple(name, types): msg = "NamedTuple('Name', [(f0, t0), (f1, t1), ...]); each t must be a type" types = [(n, _type_check(t, msg)) for n, t in types] From deb04a439259e87589f1cb639ed9924895acb632 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 14 Nov 2017 18:02:14 +0100 Subject: [PATCH 61/82] Minor simplification --- Lib/typing.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Lib/typing.py b/Lib/typing.py index 32fb7442ec876c..cfc305012f6a00 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -118,10 +118,8 @@ def _type_check(arg, msg): if isinstance(arg, str): return ForwardRef(arg) if ( - # Bare Union etc. are not valid as type arguments - _GenericAlias and isinstance(arg, _GenericAlias) and - arg.__origin__ in (Generic, _Protocol, ClassVar) or - arg in (Generic, _Protocol, ClassVar, Union, NoReturn, Optional) + isinstance(arg, _GenericAlias) and arg.__origin__ in (Generic, _Protocol, ClassVar) or + arg in (Generic, _Protocol) or isinstance(arg, _SpecialForm) and arg != Any ): raise TypeError("Plain %s is not valid as type argument" % arg) if isinstance(arg, (type, TypeVar, ForwardRef)): From b15eab8e219991e41490f9921c41a76454bb0904 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 14 Nov 2017 18:19:51 +0100 Subject: [PATCH 62/82] Minor simplifications; docstrings; formatting --- Lib/typing.py | 36 +++++++++++++++++------------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/Lib/typing.py b/Lib/typing.py index cfc305012f6a00..b1e203628757c8 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -118,8 +118,10 @@ def _type_check(arg, msg): if isinstance(arg, str): return ForwardRef(arg) if ( - isinstance(arg, _GenericAlias) and arg.__origin__ in (Generic, _Protocol, ClassVar) or - arg in (Generic, _Protocol) or isinstance(arg, _SpecialForm) and arg != Any + isinstance(arg, _GenericAlias) and + arg.__origin__ in (Generic, _Protocol, ClassVar) or + arg in (Generic, _Protocol) + or isinstance(arg, _SpecialForm) and arg != Any ): raise TypeError("Plain %s is not valid as type argument" % arg) if isinstance(arg, (type, TypeVar, ForwardRef)): @@ -367,7 +369,6 @@ def __getitem__(self, parameters): static type checkers. At runtime, Any should not be used with instance or class checks. """) - NoReturn = _SpecialForm('NoReturn', doc= """Special type indicating functions that never return. Example:: @@ -380,7 +381,6 @@ def stop() -> NoReturn: This type is invalid in other positions, e.g., ``List[NoReturn]`` will fail in static type checkers. """) - ClassVar = _SpecialForm('ClassVar', doc= """Special type construct to mark class variables. @@ -397,7 +397,6 @@ class Starship: Note that ClassVar is not a class itself, and should not be used with isinstance() or issubclass(). """) - Union = _SpecialForm('Union', doc= """Union type; Union[X, Y] means either X or Y. @@ -437,7 +436,6 @@ class Manager(Employee): pass - You cannot subclass or instantiate a union. - You can use Optional[X] as a shorthand for Union[X, None]. """) - Optional = _SpecialForm('Optional', doc= """Optional type. @@ -589,12 +587,20 @@ def __repr__(self): _sentinel = object() + def _is_dunder(attr): return attr.startswith('__') and attr.endswith('__') class _GenericAlias(_Final, _root=True): + """The central part of internal API. + This represents a generic version of type 'origin' with type arguments 'params'. + There are two kind of these aliases: user defined and special. The special ones + are wrappers around builtin collections and ABCs in collections.abc. These must + have 'name' always set. If 'inst' is False, then the alias can't be instantiated, + this is used by e.g. typing.List and typing.Dict. + """ def __init__(self, origin, params, *, name=None, inst=True, special=False): self._name = name @@ -713,7 +719,9 @@ def __subclasscheck__(self, cls): class _VariadicGenericAlias(_GenericAlias, _root=True): - + """Same as _GenericAlias above but for variadic aliases. Currently, + this is used only by special internal aliases: Tuple and Callable. + """ def __getitem__(self, params): if self._name != 'Callable' or not self._special: return self.__getitem_inner__(params) @@ -777,7 +785,6 @@ def lookup_name(mapping: Mapping[KT, VT], key: KT, default: VT) -> VT: except KeyError: return default """ - __slots__ = () def __new__(cls, *args, **kwds): @@ -813,9 +820,8 @@ def __class_getitem__(cls, params): def __init_subclass__(cls, *args, **kwargs): tvars = [] - if (not '__orig_bases__' in cls.__dict__ and Generic in cls.__bases__ and - cls.__name__ not in ('Tuple', 'Callable') or - hasattr(cls, '__orig_bases__') and Generic in cls.__orig_bases__): + if ('__orig_bases__' in cls.__dict__ and Generic in cls.__orig_bases__ or + not '__orig_bases__' in cls.__dict__ and Generic in cls.__bases__): raise TypeError("Cannot inherit from plain Generic") if '__orig_bases__' in cls.__dict__: tvars = _collect_type_vars(cls.__orig_bases__) @@ -1323,8 +1329,6 @@ def _make_nmtuple(name, types): return nm_tpl -_PY36 = sys.version_info[:2] >= (3, 6) - # attributes prohibited to set in NamedTuple class syntax _prohibited = ('__new__', '__init__', '__slots__', '__getnewargs__', '_fields', '_field_defaults', '_field_types', @@ -1338,9 +1342,6 @@ class NamedTupleMeta(type): def __new__(cls, typename, bases, ns): if ns.get('_root', False): return super().__new__(cls, typename, bases, ns) - if not _PY36: - raise TypeError("Class syntax for NamedTuple is only supported" - " in Python 3.6+") types = ns.get('__annotations__', {}) nm_tpl = _make_nmtuple(typename, types.items()) defaults = [] @@ -1395,9 +1396,6 @@ class Employee(NamedTuple): _root = True def __new__(self, typename, fields=None, **kwargs): - if kwargs and not _PY36: - raise TypeError("Keyword syntax for NamedTuple is only supported" - " in Python 3.6+") if fields is None: fields = kwargs.items() elif kwargs: From 13fa73c1bf5b95c614fcc8cb53f740bcf5c421c2 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 26 Nov 2017 21:07:09 +0100 Subject: [PATCH 63/82] Add a small FIXME --- Lib/typing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/typing.py b/Lib/typing.py index b1e203628757c8..fed1f410907113 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -688,7 +688,7 @@ def __mro_entries__(self, bases): res.append(Generic) return tuple(res) if self.__origin__ is Generic: - i = bases.index(self) + i = bases.index(self) # FIXME: class C(Generic[T], Generic[T]): pass for b in bases[i+1:]: if isinstance(b, _GenericAlias): return () From 252656ee74d02dd6fadabc613ae2fe7447a1f480 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 16 Dec 2017 21:46:49 +0100 Subject: [PATCH 64/82] Fix merge --- Objects/abstract.c | 1 - Python/bltinmodule.c | 11 ----------- 2 files changed, 12 deletions(-) diff --git a/Objects/abstract.c b/Objects/abstract.c index e21f326f56bd64..0105c5d16961e0 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -143,7 +143,6 @@ PyObject * PyObject_GetItem(PyObject *o, PyObject *key) { PyMappingMethods *m; - PyObject *meth, *result, *stack[2] = {o, key}; if (o == NULL || key == NULL) { return null_error(); diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index d08fbcdbc32dfc..e702f7c6e9e5a2 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -133,7 +133,6 @@ builtin___build_class__(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *func, *name, *bases, *mkw, *meta, *winner, *prep, *ns, *orig_bases; PyObject *cls = NULL, *cell = NULL; int isclass = 0; /* initialize to prevent gcc warning */ - int modified_bases = 0; if (nargs < 2) { PyErr_SetString(PyExc_TypeError, @@ -162,16 +161,6 @@ builtin___build_class__(PyObject *self, PyObject *const *args, Py_ssize_t nargs, return NULL; } - new_bases = update_bases(bases, args, nargs, &modified_bases); - if (new_bases == NULL) { - Py_DECREF(bases); - return NULL; - } - else { - old_bases = bases; - bases = new_bases; - } - if (kwnames == NULL) { meta = NULL; mkw = NULL; From 6b5a13c6752e60fd8e250465a74a20c07e4580ac Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 16 Dec 2017 21:58:22 +0100 Subject: [PATCH 65/82] Enable previously skipped tests --- Lib/test/test_typing.py | 22 +++++++++++----------- Lib/typing.py | 4 ++-- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 9aebda8e03450c..9b9737eea5b2c7 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -1589,7 +1589,7 @@ async def __aexit__(self, etype, eval, tb): PY36_TESTS = """ from test import ann_module, ann_module2, ann_module3 -#from typing import AsyncContextManager +from typing import AsyncContextManager class A: y: float @@ -1631,15 +1631,15 @@ class HasForeignBaseClass(mod_generics_cache.A): some_xrepr: 'XRepr' other_a: 'mod_generics_cache.A' -#async def g_with(am: AsyncContextManager[int]): -# x: int -# async with am as x: -# return x -# -#try: -# g_with(ACM()).send(None) -#except StopIteration as e: -# assert e.args[0] == 42 +async def g_with(am: AsyncContextManager[int]): + x: int + async with am as x: + return x + +try: + g_with(ACM()).send(None) +except StopIteration as e: + assert e.args[0] == 42 """ if PY36: @@ -2218,7 +2218,7 @@ def manager(): self.assertIsInstance(cm, typing.ContextManager) self.assertNotIsInstance(42, typing.ContextManager) - @skipUnless(False, "Temporary") # (ASYNCIO, 'Python 3.5 required') + @skipUnless(ASYNCIO, 'Python 3.5 required') def test_async_contextmanager(self): class NotACM: pass diff --git a/Lib/typing.py b/Lib/typing.py index fed1f410907113..486b4a5a2bb3dd 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -1230,8 +1230,8 @@ def __class_getitem__(cls, params): name='ValuesView', special=True) ContextManager = _GenericAlias(contextlib.AbstractContextManager, T_co, name='ContextManager', special=True) -#AsyncContextManager = _GenericAlias(contextlib.AbstractAsyncContextManager, T_co, -# name='AsyncContextManager') +AsyncContextManager = _GenericAlias(contextlib.AbstractAsyncContextManager, T_co, + name='AsyncContextManager', special=True) Dict = _GenericAlias(dict, (KT, VT), name='Dict', inst=False, special=True) DefaultDict = _GenericAlias(collections.defaultdict, (KT, VT), name='DefaultDict', special=True) From 5bf51d1489fd82fad245daa33469014ff042af5f Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 16 Dec 2017 22:43:14 +0100 Subject: [PATCH 66/82] Fix __all__ --- Lib/typing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/typing.py b/Lib/typing.py index 486b4a5a2bb3dd..de401da1cb6d83 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -63,7 +63,7 @@ 'Coroutine', 'Collection', 'AsyncGenerator', - # 'AsyncContextManager', + 'AsyncContextManager', # Structural checks, a.k.a. protocols. 'Reversible', From 2cf49299ad504513b82e0255198da18b3bac2519 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 16 Dec 2017 23:03:08 +0100 Subject: [PATCH 67/82] Fix tests --- Lib/dataclasses.py | 3 ++- Lib/test/test_pydoc.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 7a725dfb5208bb..df2e5d1dc3bc4b 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -388,7 +388,8 @@ def _get_field(cls, a_name, a_type): if typing is not None: # This test uses a typing internal class, but it's the best # way to test if this is a ClassVar. - if type(a_type) is typing._ClassVar: + if (type(a_type) is typing._GenericAlias and + a_type.__origin__ is typing.ClassVar): # This field is a ClassVar, so it's not a field. f._field_type = _FIELD_CLASSVAR diff --git a/Lib/test/test_pydoc.py b/Lib/test/test_pydoc.py index 1926cffba263a2..0058dceed0deba 100644 --- a/Lib/test/test_pydoc.py +++ b/Lib/test/test_pydoc.py @@ -827,7 +827,7 @@ class C(typing.Generic[T], typing.Mapping[int, str]): ... 'f\x08fo\x08oo\x08o(data: List[Any], x: int)' ' -> Iterator[Tuple[int, Any]]') self.assertEqual(pydoc.render_doc(C).splitlines()[2], - 'class C\x08C(typing.Mapping)') + 'class C\x08C(collections.abc.Mapping, typing.Generic)') def test_builtin(self): for name in ('str', 'str.translate', 'builtins.str', From eebe228a77433b45801f1347eb8d8af6c0d7aba7 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 28 Dec 2017 01:03:47 +0100 Subject: [PATCH 68/82] Add one more TODO item --- Lib/typing.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/typing.py b/Lib/typing.py index de401da1cb6d83..25d4beeffef038 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -1184,6 +1184,7 @@ def __class_getitem__(cls, params): name='Collection', special=True) Callable = _VariadicGenericAlias(collections.abc.Callable, (), name='Callable', special=True) +# TODO: Fix the interaction with -OO flag Callable.__doc__ = \ """Callable type; Callable[[int], str] is a function of (int) -> str. From 9b4911beb549b2000549a12a0a2dd7f44ce7bd28 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 11 Jan 2018 22:47:05 +0000 Subject: [PATCH 69/82] Fix __class_getitem__ calling convention --- Lib/typing.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Lib/typing.py b/Lib/typing.py index 25d4beeffef038..d892ae5f609f1b 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -821,7 +821,8 @@ def __class_getitem__(cls, params): def __init_subclass__(cls, *args, **kwargs): tvars = [] if ('__orig_bases__' in cls.__dict__ and Generic in cls.__orig_bases__ or - not '__orig_bases__' in cls.__dict__ and Generic in cls.__bases__): + '__orig_bases__' not in cls.__dict__ and Generic in cls.__bases__ + and cls.__name__ != '_Protocol'): raise TypeError("Cannot inherit from plain Generic") if '__orig_bases__' in cls.__dict__: tvars = _collect_type_vars(cls.__orig_bases__) @@ -1129,7 +1130,7 @@ def _get_protocol_attrs(self): return attrs -class _Protocol(metaclass=_ProtocolMeta): +class _Protocol(Generic, metaclass=_ProtocolMeta): """Internal base class for protocol classes. This implements a simple-minded structural issubclass check @@ -1142,7 +1143,7 @@ class _Protocol(metaclass=_ProtocolMeta): _is_protocol = True def __class_getitem__(cls, params): - return Generic.__class_getitem__(cls, params) + return super().__class_getitem__(params) # Some unconstrained type variables. These are used by the container types. From 18bdfbc39c2dc04206ae551e9f98a9beda06e986 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 11 Jan 2018 23:02:12 +0000 Subject: [PATCH 70/82] Address review comments --- Lib/test/test_typing.py | 46 ++++++++--------------------------------- Lib/typing.py | 4 ++-- 2 files changed, 11 insertions(+), 39 deletions(-) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 9b9737eea5b2c7..9f1b10eabdb594 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -28,9 +28,6 @@ from test import mod_generics_cache -PY36 = sys.version_info[:2] >= (3, 6) - - class BaseTestCase(TestCase): def assertIsSubclass(self, cls, class_or_tuple, msg=None): @@ -597,8 +594,9 @@ def test_basics(self): Y[str] with self.assertRaises(TypeError): Y[str, str] + SM1 = SimpleMapping[str, int] with self.assertRaises(TypeError): - issubclass(SimpleMapping[str, int], SimpleMapping) + issubclass(SM1, SimpleMapping) self.assertIsInstance(SimpleMapping[str, int](), SimpleMapping) def test_generic_errors(self): @@ -629,7 +627,6 @@ def test_init(self): with self.assertRaises(TypeError): Generic[T, S, T] - @skipUnless(PY36, "__init_subclass__ support required") def test_init_subclass(self): class X(typing.Generic[T]): def __init_subclass__(cls, **kwargs): @@ -1538,8 +1535,6 @@ def blah(): blah() -ASYNCIO = sys.version_info[:2] >= (3, 5) - ASYNCIO_TESTS = """ import asyncio @@ -1577,17 +1572,15 @@ async def __aexit__(self, etype, eval, tb): return None """ -if ASYNCIO: - try: - exec(ASYNCIO_TESTS) - except ImportError: - ASYNCIO = False +try: + exec(ASYNCIO_TESTS) +except ImportError: + ASYNCIO = False # multithreading is not enabled else: - # fake names for the sake of static analysis - asyncio = None - AwaitableWrapper = AsyncIteratorWrapper = ACM = object + ASYNCIO = True + +# Definitions needed for features introduced in Python 3.6 -PY36_TESTS = """ from test import ann_module, ann_module2, ann_module3 from typing import AsyncContextManager @@ -1640,15 +1633,6 @@ async def g_with(am: AsyncContextManager[int]): g_with(ACM()).send(None) except StopIteration as e: assert e.args[0] == 42 -""" - -if PY36: - exec(PY36_TESTS) -else: - # fake names for the sake of static analysis - ann_module = ann_module2 = ann_module3 = None - A = B = CSub = G = CoolEmployee = CoolEmployeeWithDefault = object - XMeth = XRepr = NoneAndForward = object gth = get_type_hints @@ -1663,14 +1647,12 @@ def test_get_type_hints_from_various_objects(self): with self.assertRaises(TypeError): gth(None) - @skipUnless(PY36, 'Python 3.6 required') def test_get_type_hints_modules(self): ann_module_type_hints = {1: 2, 'f': Tuple[int, int], 'x': int, 'y': str} self.assertEqual(gth(ann_module), ann_module_type_hints) self.assertEqual(gth(ann_module2), {}) self.assertEqual(gth(ann_module3), {}) - @skipUnless(PY36, 'Python 3.6 required') @expectedFailure def test_get_type_hints_modules_forwardref(self): # FIXME: This currently exposes a bug in typing. Cached forward references @@ -1680,7 +1662,6 @@ def test_get_type_hints_modules_forwardref(self): 'default_b': Optional[mod_generics_cache.B]} self.assertEqual(gth(mod_generics_cache), mgc_hints) - @skipUnless(PY36, 'Python 3.6 required') def test_get_type_hints_classes(self): self.assertEqual(gth(ann_module.C), # gth will find the right globalns {'y': Optional[ann_module.C]}) @@ -1703,7 +1684,6 @@ def test_get_type_hints_classes(self): 'my_inner_a2': mod_generics_cache.B.A, 'my_outer_a': mod_generics_cache.A}) - @skipUnless(PY36, 'Python 3.6 required') def test_respect_no_type_check(self): @no_type_check class NoTpCheck: @@ -1742,7 +1722,6 @@ class B: ... b.__annotations__ = {'x': 'A'} self.assertEqual(gth(b, locals()), {'x': A}) - @skipUnless(PY36, 'Python 3.6 required') def test_get_type_hints_ClassVar(self): self.assertEqual(gth(ann_module2.CV, ann_module2.__dict__), {'var': typing.ClassVar[ann_module2.CV]}) @@ -2041,7 +2020,6 @@ def test_no_generator_instantiation(self): with self.assertRaises(TypeError): typing.Generator[int, int, int]() - @skipUnless(PY36, 'Python 3.6 required') def test_async_generator(self): ns = {} exec("async def f():\n" @@ -2049,7 +2027,6 @@ def test_async_generator(self): g = ns['f']() self.assertIsSubclass(type(g), typing.AsyncGenerator) - @skipUnless(PY36, 'Python 3.6 required') def test_no_async_generator_instantiation(self): with self.assertRaises(TypeError): typing.AsyncGenerator() @@ -2127,7 +2104,6 @@ def g(): yield 0 self.assertIsSubclass(G, collections.abc.Iterable) self.assertNotIsSubclass(type(g), G) - @skipUnless(PY36, 'Python 3.6 required') def test_subclassing_async_generator(self): class G(typing.AsyncGenerator[int, int]): def asend(self, value): @@ -2322,7 +2298,6 @@ def test_namedtuple_pyversion(self): class NotYet(NamedTuple): whatever = 0 - @skipUnless(PY36, 'Python 3.6 required') def test_annotation_usage(self): tim = CoolEmployee('Tim', 9000) self.assertIsInstance(tim, CoolEmployee) @@ -2335,7 +2310,6 @@ def test_annotation_usage(self): collections.OrderedDict(name=str, cool=int)) self.assertIs(CoolEmployee._field_types, CoolEmployee.__annotations__) - @skipUnless(PY36, 'Python 3.6 required') def test_annotation_usage_with_default(self): jelle = CoolEmployeeWithDefault('Jelle') self.assertIsInstance(jelle, CoolEmployeeWithDefault) @@ -2357,7 +2331,6 @@ class NonDefaultAfterDefault(NamedTuple): y: int """) - @skipUnless(PY36, 'Python 3.6 required') def test_annotation_usage_with_methods(self): self.assertEqual(XMeth(1).double(), 2) self.assertEqual(XMeth(42).x, XMeth(42)[0]) @@ -2380,7 +2353,6 @@ def _source(self): return 'no chance for this as well' """) - @skipUnless(PY36, 'Python 3.6 required') def test_namedtuple_keyword_usage(self): LocalEmployee = NamedTuple("LocalEmployee", name=str, age=int) nick = LocalEmployee('Nick', 25) diff --git a/Lib/typing.py b/Lib/typing.py index d892ae5f609f1b..fbd64657e97d73 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -1593,9 +1593,9 @@ class io: io.__name__ = __name__ + '.io' sys.modules[io.__name__] = io -Pattern = _GenericAlias(type(stdlib_re.compile('')), AnyStr, +Pattern = _GenericAlias(stdlib_re.Pattern, AnyStr, name='Pattern', special=True) -Match = _GenericAlias(type(stdlib_re.match('', '')), AnyStr, +Match = _GenericAlias(stdlib_re.Match, AnyStr, name='Match', special=True) class re: From 3296b1b0285f9db8f721469d470363be7026adb6 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 11 Jan 2018 23:05:02 +0000 Subject: [PATCH 71/82] Minor style fix --- Lib/typing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/typing.py b/Lib/typing.py index fbd64657e97d73..c8d8c83523720a 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -121,7 +121,7 @@ def _type_check(arg, msg): isinstance(arg, _GenericAlias) and arg.__origin__ in (Generic, _Protocol, ClassVar) or arg in (Generic, _Protocol) - or isinstance(arg, _SpecialForm) and arg != Any + or isinstance(arg, _SpecialForm) and arg is not Any ): raise TypeError("Plain %s is not valid as type argument" % arg) if isinstance(arg, (type, TypeVar, ForwardRef)): From bee8838e8850d8577441229bd50906294584b9e5 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 14 Jan 2018 12:24:13 +0000 Subject: [PATCH 72/82] Some style changes; use f-strings for speed --- Lib/test/test_typing.py | 2 +- Lib/typing.py | 74 +++++++++++++++++++++-------------------- 2 files changed, 39 insertions(+), 37 deletions(-) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 9f1b10eabdb594..7de975fb3709a0 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -597,7 +597,7 @@ def test_basics(self): SM1 = SimpleMapping[str, int] with self.assertRaises(TypeError): issubclass(SM1, SimpleMapping) - self.assertIsInstance(SimpleMapping[str, int](), SimpleMapping) + self.assertIsInstance(SM1(), SimpleMapping) def test_generic_errors(self): T = TypeVar('T') diff --git a/Lib/typing.py b/Lib/typing.py index c8d8c83523720a..5ad5c70c37c839 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -1,18 +1,21 @@ """ -The typing module: - -* Imports -* Exports -* Internal helper functions -* _SpecialForm and its instances: Any, NoReturn, ClassVar, Union, Optional -* Two things that can be type arguments in addition to types: ForwardRef and TypeVar -* The central internal API: _GenericAlias, _VariadicGenericAlias -* The public counterpart of the API: Generic, Protocol (soon) -* Public functions: get_type_hints, overload, cast, no_type_check, - no_type_check_decorator -* Generic aliases for collections.abc ABCs and few additional protocols -* Special types: NewType, NamedTuple, TypedDict (soon) -* Wrapper re and io related types +The typing module: Support for gradual typing as defined by PEP 484. + +At large scale, the structure of the module is following: +* Imports and exports, all public names should be explicitelly added to __all__. +* Internal helper functions: these should never be used in code outside this module. +* _SpecialForm and its instances (special forms): Any, NoReturn, ClassVar, Union, Optional +* Two classes whose instances can be type arguments in addition to types: ForwardRef and TypeVar +* The core of internal generics API: _GenericAlias and _VariadicGenericAlias, the latter is + currently only used by Tuple and Callable. All subscripted types like X[int], Union[int, str], + etc., are instances of either of these classes. +* The public counterpart of the generics API consists of two classes: Generic and Protocol + (the latter is currently private, but will be made public after PEP 544 acceptance). +* Public helper functions: get_type_hints, overload, cast, no_type_check, + no_type_check_decorator. +* Generic aliases for collections.abc ABCs and few additional protocols. +* Special types: NewType, NamedTuple, TypedDict (may be added soon). +* Wrapper submodules for re and io related types. """ import abc @@ -105,8 +108,9 @@ def _type_check(arg, msg): """Check that the argument is a type, and return it (internal helper). - As a special case, accept None and return type(None) instead. - Also, _TypeAlias instances (e.g. Match, Pattern) are acceptable. + As a special case, accept None and return type(None) instead. Also wrap strings + into ForwardRef instances. Consider several corner cases, for example plain + special forms like Union are not valid, while Union[int, str] is OK, etc. The msg argument is a human-readable error message, e.g:: "Union[arg, ...]: arg should be a type." @@ -117,17 +121,16 @@ def _type_check(arg, msg): return type(None) if isinstance(arg, str): return ForwardRef(arg) - if ( - isinstance(arg, _GenericAlias) and - arg.__origin__ in (Generic, _Protocol, ClassVar) or - arg in (Generic, _Protocol) - or isinstance(arg, _SpecialForm) and arg is not Any - ): - raise TypeError("Plain %s is not valid as type argument" % arg) + if (isinstance(arg, _GenericAlias) and + arg.__origin__ in (Generic, _Protocol, ClassVar)): + raise TypeError(f"{arg} is not valid as type argument") + if (isinstance(arg, _SpecialForm) and arg is not Any + or arg in (Generic, _Protocol)): + raise TypeError(f"Plain {arg} is not valid as type argument") if isinstance(arg, (type, TypeVar, ForwardRef)): return arg if not callable(arg): - raise TypeError(msg + " Got %.100r." % (arg,)) + raise TypeError(f"{msg} Got {arg!r:.100}") return arg @@ -142,7 +145,7 @@ def _type_repr(obj): if isinstance(obj, type): if obj.__module__ == 'builtins': return obj.__qualname__ - return '%s.%s' % (obj.__module__, obj.__qualname__) + return f'{obj.__module__}.{obj.__qualname__}' if obj is ...: return('...') if isinstance(obj, types.FunctionType): @@ -189,12 +192,12 @@ def _check_generic(cls, parameters): This gives a nice error message in case of count mismatch. """ if not cls.__parameters__: - raise TypeError("%s is not a generic class" % repr(cls)) + raise TypeError(f"{cls} is not a generic class") alen = len(parameters) elen = len(cls.__parameters__) if alen != elen: - raise TypeError("Too %s parameters for %s; actual %s, expected %s" % - ("many" if alen > elen else "few", repr(cls), alen, elen)) + raise TypeError(f"Too {'many' if alen > elen else 'few'} parameters for {cls};" + f" actual {alen}, expected {elen}") def _remove_dups_flatten(parameters): @@ -306,7 +309,7 @@ def __new__(cls, *args, **kwds): isinstance(args[0], str) and isinstance(args[1], tuple)): # Close enough. - raise TypeError("Cannot subclass %r" % cls) + raise TypeError(f"Cannot subclass {cls}") return super().__new__(cls) def __init__(self, name, doc): @@ -328,13 +331,13 @@ def __copy__(self): return self # Special forms are immutable. def __call__(self, *args, **kwds): - raise TypeError("Cannot instantiate %r" % self) + raise TypeError(f"Cannot instantiate {self}") def __instancecheck__(self, obj): - raise TypeError("%r cannot be used with isinstance()." % self) + raise TypeError(f"{self} cannot be used with isinstance()") def __subclasscheck__(self, cls): - raise TypeError("%r cannot be used with issubclass()." % self) + raise TypeError(f"{self} cannot be used with issubclass()") @_tp_cache def __getitem__(self, parameters): @@ -451,12 +454,11 @@ class ForwardRef(_Final, _root=True): def __init__(self, arg): if not isinstance(arg, str): - raise TypeError('Forward reference must be a string -- got %r' % (arg,)) + raise TypeError(f"Forward reference must be a string -- got {arg}") try: code = compile(arg, '', 'eval') except SyntaxError: - raise SyntaxError('Forward reference must be an expression -- got %r' % - (arg,)) + raise SyntaxError(f"Forward reference must be an expression -- got {arg}") self.__forward_arg__ = arg self.__forward_code__ = code self.__forward_evaluated__ = False @@ -486,7 +488,7 @@ def __hash__(self): return hash((self.__forward_arg__, self.__forward_value__)) def __repr__(self): - return 'ForwardRef(%r)' % (self.__forward_arg__,) + return f'ForwardRef({self.__forward_arg__})' class TypeVar(_Final, _root=True): From 022e11bcae98aee480d4433c758b2884798181e0 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 14 Jan 2018 14:01:39 +0000 Subject: [PATCH 73/82] Fix remaining FIXMEs --- Lib/test/test_typing.py | 2 ++ Lib/typing.py | 11 +++++------ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 7de975fb3709a0..43d60031075458 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -608,6 +608,8 @@ def test_generic_errors(self): Generic[T][T] with self.assertRaises(TypeError): Generic[T][S] + with self.assertRaises(TypeError): + class C(Generic[T], Generic[T]): ... with self.assertRaises(TypeError): isinstance([], List[int]) with self.assertRaises(TypeError): diff --git a/Lib/typing.py b/Lib/typing.py index 5ad5c70c37c839..49f914edec9b9f 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -170,7 +170,7 @@ def _collect_type_vars(types): def _subs_tvars(tp, tvars, subs): """Substitute type variables 'tvars' with substitutions 'subs'. - These two must have same length. + These two must have the same length. """ if not isinstance(tp, _GenericAlias): return tp @@ -488,7 +488,7 @@ def __hash__(self): return hash((self.__forward_arg__, self.__forward_value__)) def __repr__(self): - return f'ForwardRef({self.__forward_arg__})' + return f'ForwardRef({self.__forward_arg__!r})' class TypeVar(_Final, _root=True): @@ -681,7 +681,7 @@ def __call__(self, *args, **kwargs): return result def __mro_entries__(self, bases): - if self._name: + if self._name: # generic version of an ABC or built-in class res = [] if self.__origin__ not in bases: res.append(self.__origin__) @@ -690,9 +690,9 @@ def __mro_entries__(self, bases): res.append(Generic) return tuple(res) if self.__origin__ is Generic: - i = bases.index(self) # FIXME: class C(Generic[T], Generic[T]): pass + i = bases.index(self) for b in bases[i+1:]: - if isinstance(b, _GenericAlias): + if isinstance(b, _GenericAlias) and b is not self: return () return (self.__origin__,) @@ -1187,7 +1187,6 @@ def __class_getitem__(cls, params): name='Collection', special=True) Callable = _VariadicGenericAlias(collections.abc.Callable, (), name='Callable', special=True) -# TODO: Fix the interaction with -OO flag Callable.__doc__ = \ """Callable type; Callable[[int], str] is a function of (int) -> str. From 73b7b0b52d5eaccdbfe3a20901bd7883e4caa1f3 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 14 Jan 2018 14:49:32 +0000 Subject: [PATCH 74/82] Add some whitespace; pospone proper solution for typing/#512 --- Lib/test/test_typing.py | 12 ++++++++++-- Lib/typing.py | 6 ++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 43d60031075458..21949ff11ae593 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -1055,12 +1055,20 @@ class C(B[int]): self.assertEqual(x.foo, 42) self.assertEqual(x.bar, 'abc') self.assertEqual(x.__dict__, {'foo': 42, 'bar': 'abc'}) - simples = [Any, Union, Tuple, Callable, ClassVar, List, typing.Iterable] - for s in simples: + samples = [Any, Union, Tuple, Callable, ClassVar] + for s in samples: for proto in range(pickle.HIGHEST_PROTOCOL + 1): z = pickle.dumps(s, proto) x = pickle.loads(z) self.assertEqual(s, x) + more_samples = [List, typing.Iterable, typing.Type] + for s in more_samples: + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + z = pickle.dumps(s, proto) + x = pickle.loads(z) + self.assertEqual(repr(s), repr(x)) # TODO: fix this + # see also comment in test_copy_and_deepcopy + # the issue is typing/#512 def test_copy_and_deepcopy(self): T = TypeVar('T') diff --git a/Lib/typing.py b/Lib/typing.py index 49f914edec9b9f..55d1dc9aa62520 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -372,6 +372,7 @@ def __getitem__(self, parameters): static type checkers. At runtime, Any should not be used with instance or class checks. """) + NoReturn = _SpecialForm('NoReturn', doc= """Special type indicating functions that never return. Example:: @@ -384,6 +385,7 @@ def stop() -> NoReturn: This type is invalid in other positions, e.g., ``List[NoReturn]`` will fail in static type checkers. """) + ClassVar = _SpecialForm('ClassVar', doc= """Special type construct to mark class variables. @@ -400,6 +402,7 @@ class Starship: Note that ClassVar is not a class itself, and should not be used with isinstance() or issubclass(). """) + Union = _SpecialForm('Union', doc= """Union type; Union[X, Y] means either X or Y. @@ -439,6 +442,7 @@ class Manager(Employee): pass - You cannot subclass or instantiate a union. - You can use Optional[X] as a shorthand for Union[X, None]. """) + Optional = _SpecialForm('Optional', doc= """Optional type. @@ -660,8 +664,6 @@ def __eq__(self, other): return False if self.__origin__ is Union and other.__origin__ is Union: return frozenset(self.__args__) == frozenset(other.__args__) - if self._special and other._special: - return True return self.__args__ == other.__args__ def __hash__(self): From 2ca8e6eb656fae3f7f1fee561fa0207145e13b4a Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 14 Jan 2018 15:31:16 +0000 Subject: [PATCH 75/82] Simplify generating special aliases for ABCs --- Lib/typing.py | 123 +++++++++++++++++++++----------------------------- 1 file changed, 52 insertions(+), 71 deletions(-) diff --git a/Lib/typing.py b/Lib/typing.py index 55d1dc9aa62520..effafcd834d457 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -607,11 +607,13 @@ class _GenericAlias(_Final, _root=True): have 'name' always set. If 'inst' is False, then the alias can't be instantiated, this is used by e.g. typing.List and typing.Dict. """ - def __init__(self, origin, params, *, - name=None, inst=True, special=False): - self._name = name + def __init__(self, origin, params, *, inst=True, special=False, name=None): self._inst = inst self._special = special + if special: + orig_name = origin.__name__ + name = orig_name[0].title() + orig_name[1:] + self._name = name if not isinstance(params, tuple): params = (params,) self.__origin__ = origin @@ -849,10 +851,10 @@ def __init_subclass__(cls, *args, **kwargs): tvarset = set(tvars) gvarset = set(gvars) if not tvarset <= gvarset: - raise TypeError( - f"Some type variables " - "({', '.join(str(t) for t in tvars if t not in gvarset)}) " - "are not listed in Generic[{', '.join(str(g) for g in gvars)}]") + s_vars = ', '.join(str(t) for t in tvars if t not in gvarset) + s_args = ', '.join(str(g) for g in gvars) + raise TypeError(f"Some type variables ({s_vars}) are" + f" not listed in Generic[{s_args}]") tvars = gvars cls.__parameters__ = tuple(tvars) @@ -1166,29 +1168,23 @@ def __class_getitem__(cls, params): # (This one *is* for export!) AnyStr = TypeVar('AnyStr', bytes, str) + # Various ABCs mimicking those in collections.abc. -Hashable = _GenericAlias(collections.abc.Hashable, (), - name='Hashable', special=True) # Not generic. -Awaitable = _GenericAlias(collections.abc.Awaitable, T_co, - name='Awaitable', special=True) -Coroutine = _GenericAlias(collections.abc.Coroutine, (T_co, T_contra, V_co), - name='Coroutine', special=True) -AsyncIterable = _GenericAlias(collections.abc.AsyncIterable, T_co, - name='AsyncIterable', special=True) -AsyncIterator = _GenericAlias(collections.abc.AsyncIterator, T_co, - name='AsyncIterator', special=True) -Iterable = _GenericAlias(collections.abc.Iterable, T_co, name='Iterable', special=True) -Iterator = _GenericAlias(collections.abc.Iterator, T_co, name='Iterator', special=True) -Reversible = _GenericAlias(collections.abc.Reversible, T_co, - name='Reversible', special=True) -Sized = _GenericAlias(collections.abc.Sized, (), - name='Sized', special=True) # Not generic. -Container = _GenericAlias(collections.abc.Container, T_co, - name='Container', special=True) -Collection = _GenericAlias(collections.abc.Collection, T_co, - name='Collection', special=True) -Callable = _VariadicGenericAlias(collections.abc.Callable, (), - name='Callable', special=True) +def _alias(origin, params, inst=True): + return _GenericAlias(origin, params, special=True, inst=inst) + +Hashable = _alias(collections.abc.Hashable, ()) # Not generic. +Awaitable = _alias(collections.abc.Awaitable, T_co) +Coroutine = _alias(collections.abc.Coroutine, (T_co, T_contra, V_co)) +AsyncIterable = _alias(collections.abc.AsyncIterable, T_co) +AsyncIterator = _alias(collections.abc.AsyncIterator, T_co) +Iterable = _alias(collections.abc.Iterable, T_co) +Iterator = _alias(collections.abc.Iterator, T_co) +Reversible = _alias(collections.abc.Reversible, T_co) +Sized = _alias(collections.abc.Sized, ()) # Not generic. +Container = _alias(collections.abc.Container, T_co) +Collection = _alias(collections.abc.Collection, T_co) +Callable = _VariadicGenericAlias(collections.abc.Callable, (), special=True) Callable.__doc__ = \ """Callable type; Callable[[int], str] is a function of (int) -> str. @@ -1199,20 +1195,15 @@ def __class_getitem__(cls, params): There is no syntax to indicate optional or keyword arguments, such function types are rarely used as callback types. """ -AbstractSet = _GenericAlias(collections.abc.Set, T_co, name='AbstractSet', special=True) -MutableSet = _GenericAlias(collections.abc.MutableSet, T, - name='MutableSet', special=True) +AbstractSet = _alias(collections.abc.Set, T_co) +MutableSet = _alias(collections.abc.MutableSet, T) # NOTE: Mapping is only covariant in the value type. -Mapping = _GenericAlias(collections.abc.Mapping, (KT, VT_co), - name='Mapping', special=True) -MutableMapping = _GenericAlias(collections.abc.MutableMapping, (KT, VT), - name='MutableMapping', special=True) -Sequence = _GenericAlias(collections.abc.Sequence, T_co, name='Sequence', special=True) -MutableSequence = _GenericAlias(collections.abc.MutableSequence, T, - name='MutableSequence', special=True) -ByteString = _GenericAlias(collections.abc.ByteString, (), - name='ByteString', special=True) # Not generic -Tuple = _VariadicGenericAlias(tuple, (), name='Tuple', inst=False, special=True) +Mapping = _alias(collections.abc.Mapping, (KT, VT_co)) +MutableMapping = _alias(collections.abc.MutableMapping, (KT, VT)) +Sequence = _alias(collections.abc.Sequence, T_co) +MutableSequence = _alias(collections.abc.MutableSequence, T) +ByteString = _alias(collections.abc.ByteString, ()) # Not generic +Tuple = _VariadicGenericAlias(tuple, (), inst=False, special=True) Tuple.__doc__ = \ """Tuple type; Tuple[X, Y] is the cross-product type of X and Y. @@ -1222,31 +1213,23 @@ def __class_getitem__(cls, params): To specify a variable-length tuple of homogeneous type, use Tuple[T, ...]. """ -List = _GenericAlias(list, T, name='List', inst=False, special=True) -Deque = _GenericAlias(collections.deque, T, name='Deque', special=True) -Set = _GenericAlias(set, T, name='Set', inst=False, special=True) -FrozenSet = _GenericAlias(frozenset, T_co, name='FrozenSet', inst=False, special=True) -MappingView = _GenericAlias(collections.abc.MappingView, T_co, - name='MappingView', special=True) -KeysView = _GenericAlias(collections.abc.KeysView, KT, name='KeysView', special=True) -ItemsView = _GenericAlias(collections.abc.ItemsView, (KT, VT_co), - name='ItemsView', special=True) -ValuesView = _GenericAlias(collections.abc.ValuesView, VT_co, - name='ValuesView', special=True) -ContextManager = _GenericAlias(contextlib.AbstractContextManager, T_co, - name='ContextManager', special=True) -AsyncContextManager = _GenericAlias(contextlib.AbstractAsyncContextManager, T_co, - name='AsyncContextManager', special=True) -Dict = _GenericAlias(dict, (KT, VT), name='Dict', inst=False, special=True) -DefaultDict = _GenericAlias(collections.defaultdict, (KT, VT), - name='DefaultDict', special=True) -Counter = _GenericAlias(collections.Counter, T, name='Counter', special=True) -ChainMap = _GenericAlias(collections.ChainMap, (KT, VT), name='ChainMap', special=True) -Generator = _GenericAlias(collections.abc.Generator, (T_co, T_contra, V_co), - name='Generator', special=True) -AsyncGenerator = _GenericAlias(collections.abc.AsyncGenerator, (T_co, T_contra), - name='AsyncGenerator', special=True) -Type = _GenericAlias(type, CT_co, name='Type', inst=False, special=True) +List = _alias(list, T, inst=False) +Deque = _alias(collections.deque, T) +Set = _alias(set, T, inst=False) +FrozenSet = _alias(frozenset, T_co, inst=False) +MappingView = _alias(collections.abc.MappingView, T_co) +KeysView = _alias(collections.abc.KeysView, KT) +ItemsView = _alias(collections.abc.ItemsView, (KT, VT_co)) +ValuesView = _alias(collections.abc.ValuesView, VT_co) +ContextManager = _alias(contextlib.AbstractContextManager, T_co) +AsyncContextManager = _alias(contextlib.AbstractAsyncContextManager, T_co) +Dict = _alias(dict, (KT, VT), inst=False) +DefaultDict = _alias(collections.defaultdict, (KT, VT)) +Counter = _alias(collections.Counter, T) +ChainMap = _alias(collections.ChainMap, (KT, VT)) +Generator = _alias(collections.abc.Generator, (T_co, T_contra, V_co)) +AsyncGenerator = _alias(collections.abc.AsyncGenerator, (T_co, T_contra)) +Type = _alias(type, CT_co, inst=False) Type.__doc__ = \ """A special construct usable to annotate class objects. @@ -1596,10 +1579,8 @@ class io: io.__name__ = __name__ + '.io' sys.modules[io.__name__] = io -Pattern = _GenericAlias(stdlib_re.Pattern, AnyStr, - name='Pattern', special=True) -Match = _GenericAlias(stdlib_re.Match, AnyStr, - name='Match', special=True) +Pattern = _alias(stdlib_re.Pattern, AnyStr) +Match = _alias(stdlib_re.Match, AnyStr) class re: """Wrapper namespace for re type aliases.""" From 5114efe90438812809cc26dd3dcd5b49a8dab702 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 14 Jan 2018 16:04:34 +0000 Subject: [PATCH 76/82] Get rid of unused and rarely used names --- Lib/typing.py | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/Lib/typing.py b/Lib/typing.py index effafcd834d457..ae73ce57f32470 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -43,11 +43,11 @@ 'Union', # ABCs (from collections.abc). - 'AbstractSet', # collections.abc.Set. - 'ByteString', # collections.abc.ByteString. + 'AbstractSet', + 'ByteString', 'Container', 'ContextManager', - 'Hashable', # collections.abc.Hashable. + 'Hashable', 'ItemsView', 'Iterable', 'Iterator', @@ -591,12 +591,6 @@ def __repr__(self): # * __args__ is a tuple of all arguments used in subscripting, # e.g., Dict[T, int].__args__ == (T, int). -_sentinel = object() - - -def _is_dunder(attr): - return attr.startswith('__') and attr.endswith('__') - class _GenericAlias(_Final, _root=True): """The central part of internal API. @@ -610,7 +604,7 @@ class _GenericAlias(_Final, _root=True): def __init__(self, origin, params, *, inst=True, special=False, name=None): self._inst = inst self._special = special - if special: + if special and name is None: orig_name = origin.__name__ name = orig_name[0].title() + orig_name[1:] self._name = name @@ -702,14 +696,16 @@ def __mro_entries__(self, bases): def __getattr__(self, attr): # We are carefull for copy and pickle. - if '__origin__' in self.__dict__ and not _is_dunder(attr): + # Also for simplicity we just don't relay all dunder names + if '__origin__' in self.__dict__ and not (attr.startswith('__') and attr.endswith('__')): return getattr(self.__origin__, attr) raise AttributeError(attr) def __setattr__(self, attr, val): - if not _is_dunder(attr) and attr not in ('_name', '_inst', '_special'): + if attr.startswith('__') and attr.endswith('__') or attr in ('_name', '_inst', '_special'): + super().__setattr__(attr, val) + else: setattr(self.__origin__, attr, val) - super().__setattr__(attr, val) def __instancecheck__(self, obj): return self.__subclasscheck__(type(obj)) From ff2b2ade45f875e2ea6b2691d9ba00e5d9b04192 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 14 Jan 2018 16:15:02 +0000 Subject: [PATCH 77/82] Add missing r converters --- Lib/typing.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Lib/typing.py b/Lib/typing.py index ae73ce57f32470..5ec4376e27f773 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -309,7 +309,7 @@ def __new__(cls, *args, **kwds): isinstance(args[0], str) and isinstance(args[1], tuple)): # Close enough. - raise TypeError(f"Cannot subclass {cls}") + raise TypeError(f"Cannot subclass {cls!r}") return super().__new__(cls) def __init__(self, name, doc): @@ -331,7 +331,7 @@ def __copy__(self): return self # Special forms are immutable. def __call__(self, *args, **kwds): - raise TypeError(f"Cannot instantiate {self}") + raise TypeError(f"Cannot instantiate {self!r}") def __instancecheck__(self, obj): raise TypeError(f"{self} cannot be used with isinstance()") @@ -458,11 +458,11 @@ class ForwardRef(_Final, _root=True): def __init__(self, arg): if not isinstance(arg, str): - raise TypeError(f"Forward reference must be a string -- got {arg}") + raise TypeError(f"Forward reference must be a string -- got {arg!r}") try: code = compile(arg, '', 'eval') except SyntaxError: - raise SyntaxError(f"Forward reference must be an expression -- got {arg}") + raise SyntaxError(f"Forward reference must be an expression -- got {arg!r}") self.__forward_arg__ = arg self.__forward_code__ = code self.__forward_evaluated__ = False From 19a7563ebd082074f7ea6e1ae954e58d3d9093e9 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 14 Jan 2018 18:29:59 +0000 Subject: [PATCH 78/82] Return the dot back --- Lib/typing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/typing.py b/Lib/typing.py index 5ec4376e27f773..0b22e44296254d 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -130,7 +130,7 @@ def _type_check(arg, msg): if isinstance(arg, (type, TypeVar, ForwardRef)): return arg if not callable(arg): - raise TypeError(f"{msg} Got {arg!r:.100}") + raise TypeError(f"{msg} Got {arg!r:.100}.") return arg From ddc9204c9bd73de0da2e6170109b506973be0c69 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 14 Jan 2018 20:18:02 +0000 Subject: [PATCH 79/82] Fix binary bool ops adter newline --- Lib/typing.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Lib/typing.py b/Lib/typing.py index 0b22e44296254d..65ef2bee468244 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -124,8 +124,8 @@ def _type_check(arg, msg): if (isinstance(arg, _GenericAlias) and arg.__origin__ in (Generic, _Protocol, ClassVar)): raise TypeError(f"{arg} is not valid as type argument") - if (isinstance(arg, _SpecialForm) and arg is not Any - or arg in (Generic, _Protocol)): + if (isinstance(arg, _SpecialForm) and arg is not Any or + arg in (Generic, _Protocol)): raise TypeError(f"Plain {arg} is not valid as type argument") if isinstance(arg, (type, TypeVar, ForwardRef)): return arg @@ -823,8 +823,8 @@ def __class_getitem__(cls, params): def __init_subclass__(cls, *args, **kwargs): tvars = [] if ('__orig_bases__' in cls.__dict__ and Generic in cls.__orig_bases__ or - '__orig_bases__' not in cls.__dict__ and Generic in cls.__bases__ - and cls.__name__ != '_Protocol'): + '__orig_bases__' not in cls.__dict__ and Generic in cls.__bases__ and + cls.__name__ != '_Protocol'): raise TypeError("Cannot inherit from plain Generic") if '__orig_bases__' in cls.__dict__: tvars = _collect_type_vars(cls.__orig_bases__) From 2a3ecb02e68726f63f05bac5b8d8cf0dc9daf8c0 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 14 Jan 2018 20:45:46 +0000 Subject: [PATCH 80/82] Fix corner case in multiple inheritance of special generics --- Lib/test/test_typing.py | 6 ++++++ Lib/typing.py | 3 ++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 21949ff11ae593..f330dfe64f2a49 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -1204,6 +1204,12 @@ class C(A[T, VT], Generic[VT, T, KT], B[KT, T]): self.assertEqual(C.__parameters__, (VT, T, KT)) + def test_multiple_inheritance_special(self): + S = TypeVar('S') + class B(Generic[S]): ... + class C(List[int], B): ... + self.assertEqual(C.__mro__, (C, list, B, Generic, object)) + def test_nested(self): G = Generic diff --git a/Lib/typing.py b/Lib/typing.py index 65ef2bee468244..6e10a2750ef09a 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -684,7 +684,8 @@ def __mro_entries__(self, bases): if self.__origin__ not in bases: res.append(self.__origin__) i = bases.index(self) - if not any(isinstance(b, _GenericAlias) for b in bases[i+1:]): + if not any(isinstance(b, _GenericAlias) or issubclass(b, Generic) + for b in bases[i+1:]): res.append(Generic) return tuple(res) if self.__origin__ is Generic: From 3e7ee8287700aac16c21bd3ac0e91e9f56b238c1 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 14 Jan 2018 22:30:59 +0000 Subject: [PATCH 81/82] Another round of review --- Lib/typing.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/Lib/typing.py b/Lib/typing.py index 6e10a2750ef09a..7ca080402e28ac 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -43,7 +43,7 @@ 'Union', # ABCs (from collections.abc). - 'AbstractSet', + 'AbstractSet', # collections.abc.Set. 'ByteString', 'Container', 'ContextManager', @@ -591,6 +591,9 @@ def __repr__(self): # * __args__ is a tuple of all arguments used in subscripting, # e.g., Dict[T, int].__args__ == (T, int). +def _is_dunder(attr): + return attr.startswith('__') and attr.endswith('__') + class _GenericAlias(_Final, _root=True): """The central part of internal API. @@ -698,12 +701,12 @@ def __mro_entries__(self, bases): def __getattr__(self, attr): # We are carefull for copy and pickle. # Also for simplicity we just don't relay all dunder names - if '__origin__' in self.__dict__ and not (attr.startswith('__') and attr.endswith('__')): + if '__origin__' in self.__dict__ and not _is_dunder(attr): return getattr(self.__origin__, attr) raise AttributeError(attr) def __setattr__(self, attr, val): - if attr.startswith('__') and attr.endswith('__') or attr in ('_name', '_inst', '_special'): + if _is_dunder(attr) or attr in ('_name', '_inst', '_special'): super().__setattr__(attr, val) else: setattr(self.__origin__, attr, val) @@ -823,9 +826,11 @@ def __class_getitem__(cls, params): def __init_subclass__(cls, *args, **kwargs): tvars = [] - if ('__orig_bases__' in cls.__dict__ and Generic in cls.__orig_bases__ or - '__orig_bases__' not in cls.__dict__ and Generic in cls.__bases__ and - cls.__name__ != '_Protocol'): + if '__orig_bases__' in cls.__dict__: + error = Generic in cls.__orig_bases__ + else: + error = Generic in cls.__bases__ and cls.__name__ != '_Protocol' + if error: raise TypeError("Cannot inherit from plain Generic") if '__orig_bases__' in cls.__dict__: tvars = _collect_type_vars(cls.__orig_bases__) From bc936e626f141cc974e547ae172d86ce699edd6d Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 14 Jan 2018 22:38:38 +0000 Subject: [PATCH 82/82] Fix refleak hunting mode (-R) --- Lib/test/test_typing.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index f330dfe64f2a49..3f24faf376292f 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -761,6 +761,8 @@ class C(collections.abc.Mapping, Generic[T]): ... self.assertIsInstance(1, C) C[int] self.assertIsInstance(1, C) + C._abc_registry.clear() + C._abc_cache.clear() # To keep refleak hunting mode clean def test_false_subclasses(self): class MyMapping(MutableMapping[str, str]): pass