Issue python#23722: Initialize __class__ from type.__new__()

The __class__ cell used by zero-argument super() is now initialized
from type.__new__ rather than __build_class__, so class methods
relying on that will now work correctly when called from metaclass
methods during class creation.

Patch by Martin Teichmann.

Lib/importlib/_bootstrap_external.py

@@ -236,7 +236,8 @@ def _write_atomic(path, data, mode=0o666):
 #     Python 3.6b1  3373 (add BUILD_STRING opcode #27078)
 #     Python 3.6b1  3375 (add SETUP_ANNOTATIONS and STORE_ANNOTATION opcodes
 #                         #27985)
-#     Python 3.6a1  3376 (simplify CALL_FUNCTIONs & BUILD_MAP_UNPACK_WITH_CALL)
+#     Python 3.6b1  3376 (simplify CALL_FUNCTIONs & BUILD_MAP_UNPACK_WITH_CALL)
+#     Python 3.6b1  3377 (set __class__ cell from type.__new__ #23722)
 #
 # MAGIC must change whenever the bytecode emitted by the compiler may no
 # longer be understood by older implementations of the eval loop (usually
@@ -245,7 +246,7 @@ def _write_atomic(path, data, mode=0o666):
 # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array
 # in PC/launcher.c must also be updated.
 
-MAGIC_NUMBER = (3376).to_bytes(2, 'little') + b'\r\n'
+MAGIC_NUMBER = (3377).to_bytes(2, 'little') + b'\r\n'
 _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little')  # For import.c
 
 _PYCACHE = '__pycache__'

Lib/test/test_super.py

@@ -143,6 +143,87 @@ def f():
                 return __class__
         self.assertIs(X.f(), X)
 
+    def test___class___new(self):
+        test_class = None
+
+        class Meta(type):
+            def __new__(cls, name, bases, namespace):
+                nonlocal test_class
+                self = super().__new__(cls, name, bases, namespace)
+                test_class = self.f()
+                return self
+
+        class A(metaclass=Meta):
+            @staticmethod
+            def f():
+                return __class__
+
+        self.assertIs(test_class, A)
+
+    def test___class___delayed(self):
+        test_namespace = None
+
+        class Meta(type):
+            def __new__(cls, name, bases, namespace):
+                nonlocal test_namespace
+                test_namespace = namespace
+                return None
+
+        class A(metaclass=Meta):
+            @staticmethod
+            def f():
+                return __class__
+
+        self.assertIs(A, None)
+
+        B = type("B", (), test_namespace)
+        self.assertIs(B.f(), B)
+
+    def test___class___mro(self):
+        test_class = None
+
+        class Meta(type):
+            def mro(self):
+                # self.f() doesn't work yet...
+                self.__dict__["f"]()
+                return super().mro()
+
+        class A(metaclass=Meta):
+            def f():
+                nonlocal test_class
+                test_class = __class__
+
+        self.assertIs(test_class, A)
+
+    def test___classcell___deleted(self):
+        class Meta(type):
+            def __new__(cls, name, bases, namespace):
+                del namespace['__classcell__']
+                return super().__new__(cls, name, bases, namespace)
+
+        class A(metaclass=Meta):
+            @staticmethod
+            def f():
+                __class__
+
+        with self.assertRaises(NameError):
+            A.f()
+
+    def test___classcell___reset(self):
+        class Meta(type):
+            def __new__(cls, name, bases, namespace):
+                namespace['__classcell__'] = 0
+                return super().__new__(cls, name, bases, namespace)
+
+        class A(metaclass=Meta):
+            @staticmethod
+            def f():
+                __class__
+
+        with self.assertRaises(NameError):
+            A.f()
+        self.assertEqual(A.__classcell__, 0)
+
     def test_obscure_super_errors(self):
         def f():
             super()

Misc/NEWS

@@ -10,6 +10,11 @@ What's New in Python 3.6.0 beta 1
 Core and Builtins
 -----------------
 
+- Issue #23722: The __class__ cell used by zero-argument super() is now
+  initialized from type.__new__ rather than __build_class__, so class methods
+  relying on that will now work correctly when called from metaclass methods
+  during class creation. Patch by Martin Teichmann.
+
 - Issue #25221: Fix corrupted result from PyLong_FromLong(0) when Python
   is compiled with NSMALLPOSINTS = 0.
 

Objects/typeobject.c

@@ -2285,14 +2285,15 @@ static PyObject *
 type_new(PyTypeObject *metatype, PyObject *args, PyObject *kwds)
 {
     PyObject *name, *bases = NULL, *orig_dict, *dict = NULL;
-    PyObject *qualname, *slots = NULL, *tmp, *newslots;
+    PyObject *qualname, *slots = NULL, *tmp, *newslots, *cell;
     PyTypeObject *type = NULL, *base, *tmptype, *winner;
     PyHeapTypeObject *et;
     PyMemberDef *mp;
     Py_ssize_t i, nbases, nslots, slotoffset, name_size;
     int j, may_add_dict, may_add_weak, add_dict, add_weak;
     _Py_IDENTIFIER(__qualname__);
     _Py_IDENTIFIER(__slots__);
+    _Py_IDENTIFIER(__classcell__);
 
     assert(args != NULL && PyTuple_Check(args));
     assert(kwds == NULL || PyDict_Check(kwds));
@@ -2559,7 +2560,7 @@ type_new(PyTypeObject *metatype, PyObject *args, PyObject *kwds)
     }
     et->ht_qualname = qualname ? qualname : et->ht_name;
     Py_INCREF(et->ht_qualname);
-    if (qualname != NULL && PyDict_DelItem(dict, PyId___qualname__.object) < 0)
+    if (qualname != NULL && _PyDict_DelItemId(dict, &PyId___qualname__) < 0)
         goto error;
 
     /* Set tp_doc to a copy of dict['__doc__'], if the latter is there
@@ -2685,6 +2686,14 @@ type_new(PyTypeObject *metatype, PyObject *args, PyObject *kwds)
     else
         type->tp_free = PyObject_Del;
 
+    /* store type in class' cell */
+    cell = _PyDict_GetItemId(dict, &PyId___classcell__);
+    if (cell != NULL && PyCell_Check(cell)) {
+        PyCell_Set(cell, (PyObject *) type);
+        _PyDict_DelItemId(dict, &PyId___classcell__);
+        PyErr_Clear();
+    }
+
     /* Initialize the rest */
     if (PyType_Ready(type) < 0)
         goto error;

PC/launcher.c

@@ -1089,7 +1089,7 @@ static PYC_MAGIC magic_values[] = {
     { 3190, 3230, L"3.3" },
     { 3250, 3310, L"3.4" },
     { 3320, 3351, L"3.5" },
-    { 3360, 3375, L"3.6" },
+    { 3360, 3379, L"3.6" },
     { 0 }
 };
 

Python/bltinmodule.c

@@ -54,7 +54,7 @@ _Py_IDENTIFIER(stderr);
 static PyObject *
 builtin___build_class__(PyObject *self, PyObject *args, PyObject *kwds)
 {
-    PyObject *func, *name, *bases, *mkw, *meta, *winner, *prep, *ns, *cell;
+    PyObject *func, *name, *bases, *mkw, *meta, *winner, *prep, *ns, *none;
     PyObject *cls = NULL;
     Py_ssize_t nargs;
     int isclass = 0;   /* initialize to prevent gcc warning */
@@ -167,15 +167,13 @@ builtin___build_class__(PyObject *self, PyObject *args, PyObject *kwds)
         Py_DECREF(bases);
         return NULL;
     }
-    cell = PyEval_EvalCodeEx(PyFunction_GET_CODE(func), PyFunction_GET_GLOBALS(func), ns,
+    none = PyEval_EvalCodeEx(PyFunction_GET_CODE(func), PyFunction_GET_GLOBALS(func), ns,
                              NULL, 0, NULL, 0, NULL, 0, NULL,
                              PyFunction_GET_CLOSURE(func));
-    if (cell != NULL) {
+    if (none != NULL) {
         PyObject *margs[3] = {name, bases, ns};
         cls = _PyObject_FastCallDict(meta, margs, 3, mkw);
-        if (cls != NULL && PyCell_Check(cell))
-            PyCell_Set(cell, cls);
-        Py_DECREF(cell);
+        Py_DECREF(none);
     }
     Py_DECREF(ns);
     Py_DECREF(meta);

Python/compile.c

@@ -1968,7 +1968,7 @@ compiler_class(struct compiler *c, stmt_ty s)
             return 0;
         }
         if (c->u->u_ste->ste_needs_class_closure) {
-            /* return the (empty) __class__ cell */
+            /* store __classcell__ into class namespace */
             str = PyUnicode_InternFromString("__class__");
             if (str == NULL) {
                 compiler_exit_scope(c);
@@ -1981,15 +1981,20 @@ compiler_class(struct compiler *c, stmt_ty s)
                 return 0;
             }
             assert(i == 0);
-            /* Return the cell where to store __class__ */
+
             ADDOP_I(c, LOAD_CLOSURE, i);
+            str = PyUnicode_InternFromString("__classcell__");
+            if (!str || !compiler_nameop(c, str, Store)) {
+                Py_XDECREF(str);
+                compiler_exit_scope(c);
+                return 0;
+            }
+            Py_DECREF(str);
         }
         else {
+            /* This happens when nobody references the cell. */
             assert(PyDict_Size(c->u->u_cellvars) == 0);
-            /* This happens when nobody references the cell. Return None. */
-            ADDOP_O(c, LOAD_CONST, Py_None, consts);
         }
-        ADDOP_IN_SCOPE(c, RETURN_VALUE);
         /* create the code object */
         co = assemble(c, 1);
     }