bpo-35713: Reorganize sys module initialization

Reorganize Python initialization to get working exceptions and
sys.stderr earlier. Changes:

* Add _PySys_SetPreliminaryStderr(). Preliminary sys.stderr is now
  set earlier to get an usable sys.stderr ealier.
* Split _PyExc_Init(): create a new _PyBuiltins_AddExceptions()
  function.
* Call _PyExc_Init() earlier in _Py_InitializeCore_impl()
  and new_interpreter() to get working exceptions earlier.
* Split _Py_InitializeCore_impl() into subfunctions: add multiple
  pycore_init_xxx() functions
* Move code into _Py_Initialize_ReconfigureCore() to be able to call
  it from _Py_InitializeCore().
* Rename _Py_ReadyTypes() to _PyTypes_Init().
* Rename _PySys_BeginInit() to _PySys_InitCore()
* Rename _PySys_EndInit() to _PySys_InitMain()
* Add _PySys_Create(). It calls _PySys_InitCore() which becomes
  private.
* Misc code cleanup

Include/internal/pycore_pylifecycle.h

@@ -27,16 +27,19 @@ extern int _PyLong_Init(void);
 extern _PyInitError _PyFaulthandler_Init(int enable);
 extern int _PyTraceMalloc_Init(int enable);
 extern PyObject * _PyBuiltin_Init(void);
-extern _PyInitError _PySys_BeginInit(PyObject **sysmod);
-extern int _PySys_EndInit(PyObject *sysdict, PyInterpreterState *interp);
+extern _PyInitError _PySys_Create(
+    PyInterpreterState *interp,
+    PyObject **sysmod_p);
+extern _PyInitError _PySys_SetPreliminaryStderr(PyObject *sysdict);
+extern int _PySys_InitMain(PyInterpreterState *interp);
 extern _PyInitError _PyImport_Init(PyInterpreterState *interp);
 extern _PyInitError _PyExc_Init(void);
 extern _PyInitError _PyBuiltins_AddExceptions(PyObject * bltinmod);
 extern _PyInitError _PyImportHooks_Init(void);
 extern int _PyFloat_Init(void);
 extern _PyInitError _Py_HashRandomization_Init(const _PyCoreConfig *);
 
-extern _PyInitError _Py_ReadyTypes(void);
+extern _PyInitError _PyTypes_Init(void);
 
 /* Various internal finalizers */
 

Objects/object.c

@@ -1717,7 +1717,7 @@ PyObject _Py_NotImplementedStruct = {
 };
 
 _PyInitError
-_Py_ReadyTypes(void)
+_PyTypes_Init(void)
 {
 #define INIT_TYPE(TYPE, NAME) \
     do { \

Python/pylifecycle.c

@@ -589,7 +589,7 @@ pycore_create_interpreter(const _PyCoreConfig *core_config,
 static _PyInitError
 pycore_init_types(void)
 {
-    _PyInitError err = _Py_ReadyTypes();
+    _PyInitError err = _PyTypes_Init();
     if (_Py_INIT_FAILED(err)) {
         return err;
     }
@@ -623,46 +623,6 @@ pycore_init_types(void)
 }
 
 
-static _PyInitError
-pycore_init_sys(PyInterpreterState *interp, PyObject **sysmod_p)
-{
-    PyObject *modules = PyDict_New();
-    if (modules == NULL)
-        return _Py_INIT_ERR("can't make modules dictionary");
-    interp->modules = modules;
-
-    PyObject *sysmod;
-    _PyInitError err = _PySys_BeginInit(&sysmod);
-    if (_Py_INIT_FAILED(err)) {
-        return err;
-    }
-    *sysmod_p = sysmod;
-
-    interp->sysdict = PyModule_GetDict(sysmod);
-    if (interp->sysdict == NULL) {
-        return _Py_INIT_ERR("can't initialize sys dict");
-    }
-
-    Py_INCREF(interp->sysdict);
-    PyDict_SetItemString(interp->sysdict, "modules", modules);
-    _PyImport_FixupBuiltin(sysmod, "sys", modules);
-
-    /* Set up a preliminary stderr printer until we have enough
-       infrastructure for the io module in place.
-
-       Use UTF-8/surrogateescape and ignore EAGAIN errors. */
-    PyObject *pstderr = PyFile_NewStdPrinter(fileno(stderr));
-    if (pstderr == NULL) {
-        return _Py_INIT_ERR("can't set preliminary stderr");
-    }
-    _PySys_SetObjectId(&PyId_stderr, pstderr);
-    PySys_SetObject("__stderr__", pstderr);
-    Py_DECREF(pstderr);
-
-    return _Py_INIT_OK();
-}
-
-
 static _PyInitError
 pycore_init_builtins(PyInterpreterState *interp)
 {
@@ -746,7 +706,7 @@ _Py_InitializeCore_impl(PyInterpreterState **interp_p,
     }
 
     PyObject *sysmod;
-    err = pycore_init_sys(interp, &sysmod);
+    err = _PySys_Create(interp, &sysmod);
     if (_Py_INIT_FAILED(err)) {
         return err;
     }
@@ -887,7 +847,7 @@ _Py_InitializeMainInterpreter(PyInterpreterState *interp,
         return _Py_INIT_ERR("can't initialize time");
     }
 
-    if (_PySys_EndInit(interp->sysdict, interp) < 0) {
+    if (_PySys_InitMain(interp) < 0) {
         return _Py_INIT_ERR("can't finish initializing sys");
     }
 
@@ -1376,7 +1336,9 @@ new_interpreter(PyThreadState **tstate_p)
             goto handle_error;
         Py_INCREF(interp->sysdict);
         PyDict_SetItemString(interp->sysdict, "modules", modules);
-        _PySys_EndInit(interp->sysdict, interp);
+        if (_PySys_InitMain(interp) < 0) {
+            return _Py_INIT_ERR("can't finish initializing sys");
+        }
     }
     else if (PyErr_Occurred()) {
         goto handle_error;
@@ -1399,15 +1361,10 @@ new_interpreter(PyThreadState **tstate_p)
             return err;
         }
 
-        /* Set up a preliminary stderr printer until we have enough
-           infrastructure for the io module in place. */
-        PyObject *pstderr = PyFile_NewStdPrinter(fileno(stderr));
-        if (pstderr == NULL) {
-            return _Py_INIT_ERR("can't set preliminary stderr");
+        err = _PySys_SetPreliminaryStderr(interp->sysdict);
+        if (_Py_INIT_FAILED(err)) {
+            return err;
         }
-        _PySys_SetObjectId(&PyId_stderr, pstderr);
-        PySys_SetObject("__stderr__", pstderr);
-        Py_DECREF(pstderr);
 
         err = _PyImportHooks_Init();
         if (_Py_INIT_FAILED(err)) {

Python/sysmodule.c

@@ -2368,19 +2368,12 @@ static struct PyModuleDef sysmodule = {
         }                                                  \
     } while (0)
 
-
-_PyInitError
-_PySys_BeginInit(PyObject **sysmod)
+static _PyInitError
+_PySys_InitCore(PyObject *sysdict)
 {
-    PyObject *m, *sysdict, *version_info;
+    PyObject *version_info;
     int res;
 
-    m = _PyModule_CreateInitialized(&sysmodule, PYTHON_API_VERSION);
-    if (m == NULL) {
-        return _Py_INIT_ERR("failed to create a module object");
-    }
-    sysdict = PyModule_GetDict(m);
-
     /* stdin/stdout/stderr are set in pylifecycle.c */
 
     SET_SYS_FROM_STRING_BORROW("__displayhook__",
@@ -2508,9 +2501,6 @@ _PySys_BeginInit(PyObject **sysmod)
     if (PyErr_Occurred()) {
         goto err_occurred;
     }
-
-    *sysmod = m;
-
     return _Py_INIT_OK();
 
 type_init_failed:
@@ -2536,8 +2526,9 @@ _PySys_BeginInit(PyObject **sysmod)
     } while (0)
 
 int
-_PySys_EndInit(PyObject *sysdict, PyInterpreterState *interp)
+_PySys_InitMain(PyInterpreterState *interp)
 {
+    PyObject *sysdict = interp->sysdict;
     const _PyCoreConfig *core_config = &interp->core_config;
     const _PyMainInterpreterConfig *config = &interp->config;
     int res;
@@ -2552,17 +2543,16 @@ _PySys_EndInit(PyObject *sysdict, PyInterpreterState *interp)
 
 #define COPY_LIST(KEY, ATTR) \
     do { \
-        assert(PyList_Check(config->ATTR)); \
-        PyObject *list = PyList_GetSlice(config->ATTR, \
-                                         0, PyList_GET_SIZE(config->ATTR)); \
+        assert(PyList_Check(ATTR)); \
+        PyObject *list = PyList_GetSlice(ATTR, 0, PyList_GET_SIZE(ATTR)); \
         if (list == NULL) { \
             return -1; \
         } \
         SET_SYS_FROM_STRING_BORROW(KEY, list); \
         Py_DECREF(list); \
     } while (0)
 
-    COPY_LIST("path", module_search_path);
+    COPY_LIST("path", config->module_search_path);
 
     SET_SYS_FROM_STRING_BORROW("executable", config->executable);
     SET_SYS_FROM_STRING_BORROW("prefix", config->prefix);
@@ -2580,7 +2570,7 @@ _PySys_EndInit(PyObject *sysdict, PyInterpreterState *interp)
         SET_SYS_FROM_STRING_BORROW("argv", config->argv);
     }
     if (config->warnoptions != NULL) {
-        COPY_LIST("warnoptions", warnoptions);
+        COPY_LIST("warnoptions", config->warnoptions);
     }
     if (config->xoptions != NULL) {
         PyObject *dict = PyDict_Copy(config->xoptions);
@@ -2631,6 +2621,77 @@ _PySys_EndInit(PyObject *sysdict, PyInterpreterState *interp)
 #undef SET_SYS_FROM_STRING_BORROW
 #undef SET_SYS_FROM_STRING_INT_RESULT
 
+
+/* Set up a preliminary stderr printer until we have enough
+   infrastructure for the io module in place.
+
+   Use UTF-8/surrogateescape and ignore EAGAIN errors. */
+_PyInitError
+_PySys_SetPreliminaryStderr(PyObject *sysdict)
+{
+    PyObject *pstderr = PyFile_NewStdPrinter(fileno(stderr));
+    if (pstderr == NULL) {
+        goto error;
+    }
+    if (_PyDict_SetItemId(sysdict, &PyId_stderr, pstderr) < 0) {
+        goto error;
+    }
+    if (PyDict_SetItemString(sysdict, "__stderr__", pstderr) < 0) {
+        goto error;
+    }
+    Py_DECREF(pstderr);
+    return _Py_INIT_OK();
+
+error:
+    Py_XDECREF(pstderr);
+    return _Py_INIT_ERR("can't set preliminary stderr");
+}
+
+
+/* Create sys module without all attributes: _PySys_InitMain() should be called
+   later to add remaining attributes. */
+_PyInitError
+_PySys_Create(PyInterpreterState *interp, PyObject **sysmod_p)
+{
+    PyObject *modules = PyDict_New();
+    if (modules == NULL) {
+        return _Py_INIT_ERR("can't make modules dictionary");
+    }
+    interp->modules = modules;
+
+    PyObject *sysmod = _PyModule_CreateInitialized(&sysmodule, PYTHON_API_VERSION);
+    if (sysmod == NULL) {
+        return _Py_INIT_ERR("failed to create a module object");
+    }
+
+    PyObject *sysdict = PyModule_GetDict(sysmod);
+    if (sysdict == NULL) {
+        return _Py_INIT_ERR("can't initialize sys dict");
+    }
+    Py_INCREF(sysdict);
+    interp->sysdict = sysdict;
+
+    if (PyDict_SetItemString(sysdict, "modules", interp->modules) < 0) {
+        return _Py_INIT_ERR("can't initialize sys module");
+    }
+
+    _PyInitError err = _PySys_SetPreliminaryStderr(sysdict);
+    if (_Py_INIT_FAILED(err)) {
+        return err;
+    }
+
+    err = _PySys_InitCore(sysdict);
+    if (_Py_INIT_FAILED(err)) {
+        return err;
+    }
+
+    _PyImport_FixupBuiltin(sysmod, "sys", interp->modules);
+
+    *sysmod_p = sysmod;
+    return _Py_INIT_OK();
+}
+
+
 static PyObject *
 makepathobject(const wchar_t *path, wchar_t delim)
 {