Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

bpo-36763: Fix encoding/locale tests in test_embed #13443

Merged
merged 1 commit into from
May 20, 2019
Merged

bpo-36763: Fix encoding/locale tests in test_embed #13443

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
190 changes: 103 additions & 87 deletions Lib/test/test_embed.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -296,13 +296,16 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
})
PYTHON_PRE_CONFIG = dict(DEFAULT_PRE_CONFIG,
parse_argv=1,
coerce_c_locale=GET_DEFAULT_CONFIG,
utf8_mode=GET_DEFAULT_CONFIG,
)
ISOLATED_PRE_CONFIG = dict(DEFAULT_PRE_CONFIG,
configure_locale=0,
isolated=1,
use_environment=0,
utf8_mode=0,
dev_mode=0,
coerce_c_locale=0,
)

COPY_PRE_CONFIG = [
Expand Down Expand Up @@ -435,6 +438,8 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
('Py_LegacyWindowsStdioFlag', 'legacy_windows_stdio'),
))

EXPECTED_CONFIG = None

def main_xoptions(self, xoptions_list):
xoptions = {}
for opt in xoptions_list:
Expand All @@ -445,37 +450,15 @@ def main_xoptions(self, xoptions_list):
xoptions[opt] = True
return xoptions

def get_expected_config(self, expected_preconfig, expected, env, api,
add_path=None):
if api == CONFIG_INIT_PYTHON:
default_config = self.PYTHON_CORE_CONFIG
elif api == CONFIG_INIT_ISOLATED:
default_config = self.ISOLATED_CORE_CONFIG
else:
default_config = self.DEFAULT_CORE_CONFIG
expected = dict(default_config, **expected)
expected['_config_init'] = api

def _get_expected_config(self, env):
code = textwrap.dedent('''
import json
import sys
import _testinternalcapi

configs = _testinternalcapi.get_configs()
core_config = configs['core_config']
data = {
'stdio_encoding': sys.stdout.encoding,
'stdio_errors': sys.stdout.errors,
'prefix': sys.prefix,
'base_prefix': sys.base_prefix,
'exec_prefix': sys.exec_prefix,
'base_exec_prefix': sys.base_exec_prefix,
'filesystem_encoding': sys.getfilesystemencoding(),
'filesystem_errors': sys.getfilesystemencodeerrors(),
'module_search_paths': core_config['module_search_paths'],
}

data = json.dumps(data)

data = json.dumps(configs)
data = data.encode('utf-8')
sys.stdout.buffer.write(data)
sys.stdout.buffer.flush()
Expand All @@ -484,10 +467,6 @@ def get_expected_config(self, expected_preconfig, expected, env, api,
# Use -S to not import the site module: get the proper configuration
# when test_embed is run from a venv (bpo-35313)
args = [sys.executable, '-S', '-c', code]
env = dict(env)
if not expected['isolated']:
env['PYTHONCOERCECLOCALE'] = '0'
env['PYTHONUTF8'] = '0'
proc = subprocess.run(args, env=env,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
Expand All @@ -496,10 +475,46 @@ def get_expected_config(self, expected_preconfig, expected, env, api,
f"stdout={proc.stdout!r} stderr={proc.stderr!r}")
stdout = proc.stdout.decode('utf-8')
try:
config = json.loads(stdout)
return json.loads(stdout)
except json.JSONDecodeError:
self.fail(f"fail to decode stdout: {stdout!r}")

def get_expected_config(self, expected_preconfig, expected, env, api,
add_path=None):
cls = self.__class__
if cls.EXPECTED_CONFIG is None:
cls.EXPECTED_CONFIG = self._get_expected_config(env)
configs = {key: dict(value)
for key, value in self.EXPECTED_CONFIG.items()}

pre_config = configs['pre_config']
for key, value in expected_preconfig.items():
if value is self.GET_DEFAULT_CONFIG:
expected_preconfig[key] = pre_config[key]

if not expected_preconfig['configure_locale'] or api == CONFIG_INIT:
# there is no easy way to get the locale encoding before
# setlocale(LC_CTYPE, "") is called: don't test encodings
for key in ('filesystem_encoding', 'filesystem_errors',
'stdio_encoding', 'stdio_errors'):
expected[key] = self.IGNORE_CONFIG

if not expected_preconfig['configure_locale']:
# UTF-8 Mode depends on the locale. There is no easy way
# to guess if UTF-8 Mode will be enabled or not if the locale
# is not configured.
expected_preconfig['utf8_mode'] = self.IGNORE_CONFIG

if expected_preconfig['utf8_mode'] == 1:
if expected['filesystem_encoding'] is self.GET_DEFAULT_CONFIG:
expected['filesystem_encoding'] = 'utf-8'
if expected['filesystem_errors'] is self.GET_DEFAULT_CONFIG:
expected['filesystem_errors'] = self.UTF8_MODE_ERRORS
if expected['stdio_encoding'] is self.GET_DEFAULT_CONFIG:
expected['stdio_encoding'] = 'utf-8'
if expected['stdio_errors'] is self.GET_DEFAULT_CONFIG:
expected['stdio_errors'] = 'surrogateescape'

if expected['executable'] is self.GET_DEFAULT_CONFIG:
if sys.platform == 'win32':
expected['executable'] = self.test_exe
Expand All @@ -511,24 +526,28 @@ def get_expected_config(self, expected_preconfig, expected, env, api,
if expected['program_name'] is self.GET_DEFAULT_CONFIG:
expected['program_name'] = './_testembed'

core_config = configs['core_config']
for key, value in expected.items():
if value is self.GET_DEFAULT_CONFIG:
expected[key] = config[key]
expected[key] = core_config[key]

prepend_path = expected['module_search_path_env']
if prepend_path is not None:
expected['module_search_paths'] = [prepend_path, *expected['module_search_paths']]
if add_path is not None:
expected['module_search_paths'].append(add_path)

if not expected_preconfig['configure_locale']:
# there is no easy way to get the locale encoding before
# setlocale(LC_CTYPE, "") is called: don't test encodings
for key in ('filesystem_encoding', 'filesystem_errors',
'stdio_encoding', 'stdio_errors'):
expected[key] = self.IGNORE_CONFIG
expected['module_search_paths'] = [*expected['module_search_paths'], add_path]

return expected
for key in self.COPY_PRE_CONFIG:
if key not in expected_preconfig:
expected_preconfig[key] = expected[key]

def check_pre_config(self, config, expected):
self.assertEqual(config['pre_config'], expected)
pre_config = dict(config['pre_config'])
for key, value in list(expected.items()):
if value is self.IGNORE_CONFIG:
del pre_config[key]
del expected[key]
self.assertEqual(pre_config, expected)

def check_core_config(self, config, expected):
core_config = dict(config['core_config'])
Expand Down Expand Up @@ -567,10 +586,6 @@ def check_config(self, testname, expected_config=None, expected_preconfig=None,
for key in list(env):
if key.startswith('PYTHON'):
del env[key]
# Disable C locale coercion and UTF-8 mode to not depend
# on the current locale
env['PYTHONCOERCECLOCALE'] = '0'
env['PYTHONUTF8'] = '0'

if api == CONFIG_INIT_ISOLATED:
default_preconfig = self.ISOLATED_PRE_CONFIG
Expand All @@ -583,12 +598,19 @@ def check_config(self, testname, expected_config=None, expected_preconfig=None,
expected_preconfig = dict(default_preconfig, **expected_preconfig)
if expected_config is None:
expected_config = {}
expected_config = self.get_expected_config(expected_preconfig,
expected_config, env,
api, add_path)
for key in self.COPY_PRE_CONFIG:
if key not in expected_preconfig:
expected_preconfig[key] = expected_config[key]

if api == CONFIG_INIT_PYTHON:
default_config = self.PYTHON_CORE_CONFIG
elif api == CONFIG_INIT_ISOLATED:
default_config = self.ISOLATED_CORE_CONFIG
else:
default_config = self.DEFAULT_CORE_CONFIG
expected_config = dict(default_config, **expected_config)
expected_config['_config_init'] = api

self.get_expected_config(expected_preconfig,
expected_config, env,
api, add_path)

out, err = self.run_embedded_interpreter(testname, env=env)
if stderr is None and not expected_config['verbose']:
Expand Down Expand Up @@ -624,10 +646,6 @@ def test_init_global_config(self):
'quiet': 1,
'buffered_stdio': 0,

'stdio_encoding': 'utf-8',
'stdio_errors': 'surrogateescape',
'filesystem_encoding': 'utf-8',
'filesystem_errors': self.UTF8_MODE_ERRORS,
'user_site_directory': 0,
'pathconfig_warnings': 0,
}
Expand All @@ -650,8 +668,6 @@ def test_init_from_config(self):

'stdio_encoding': 'iso8859-1',
'stdio_errors': 'replace',
'filesystem_encoding': 'utf-8',
'filesystem_errors': self.UTF8_MODE_ERRORS,

'pycache_prefix': 'conf_pycache_prefix',
'program_name': './conf_program_name',
Expand Down Expand Up @@ -679,43 +695,42 @@ def test_init_from_config(self):
}
self.check_config("init_from_config", config, preconfig)

INIT_ENV_PRECONFIG = {
'allocator': PYMEM_ALLOCATOR_MALLOC,
}
INIT_ENV_CONFIG = {
'use_hash_seed': 1,
'hash_seed': 42,
'tracemalloc': 2,
'import_time': 1,
'malloc_stats': 1,
'inspect': 1,
'optimization_level': 2,
'pycache_prefix': 'env_pycache_prefix',
'write_bytecode': 0,
'verbose': 1,
'buffered_stdio': 0,
'stdio_encoding': 'iso8859-1',
'stdio_errors': 'replace',
'user_site_directory': 0,
'faulthandler': 1,
}

def test_init_env(self):
self.check_config("init_env", self.INIT_ENV_CONFIG, self.INIT_ENV_PRECONFIG)
preconfig = {
'allocator': PYMEM_ALLOCATOR_MALLOC,
}
config = {
'use_hash_seed': 1,
'hash_seed': 42,
'tracemalloc': 2,
'import_time': 1,
'malloc_stats': 1,
'inspect': 1,
'optimization_level': 2,
'module_search_path_env': '/my/path',
'pycache_prefix': 'env_pycache_prefix',
'write_bytecode': 0,
'verbose': 1,
'buffered_stdio': 0,
'stdio_encoding': 'iso8859-1',
'stdio_errors': 'replace',
'user_site_directory': 0,
'faulthandler': 1,
'warnoptions': ['EnvVar'],
}
self.check_config("init_env", config, preconfig)

def test_init_env_dev_mode(self):
preconfig = dict(self.INIT_ENV_PRECONFIG,
allocator=PYMEM_ALLOCATOR_DEBUG)
config = dict(self.INIT_ENV_CONFIG,
dev_mode=1,
preconfig = dict(allocator=PYMEM_ALLOCATOR_DEBUG)
config = dict(dev_mode=1,
faulthandler=1,
warnoptions=['default'])
self.check_config("init_env_dev_mode", config, preconfig)

def test_init_env_dev_mode_alloc(self):
preconfig = dict(self.INIT_ENV_PRECONFIG,
allocator=PYMEM_ALLOCATOR_MALLOC)
config = dict(self.INIT_ENV_CONFIG,
dev_mode=1,
preconfig = dict(allocator=PYMEM_ALLOCATOR_MALLOC)
config = dict(dev_mode=1,
faulthandler=1,
warnoptions=['default'])
self.check_config("init_env_dev_mode_alloc", config, preconfig)

Expand Down Expand Up @@ -800,6 +815,7 @@ def test_init_dont_configure_locale(self):
# _PyPreConfig.configure_locale=0
preconfig = {
'configure_locale': 0,
'coerce_c_locale': 0,
}
self.check_config("init_dont_configure_locale", {}, preconfig,
api=CONFIG_INIT_PYTHON)
Expand Down
19 changes: 10 additions & 9 deletions Programs/_testembed.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -568,7 +568,7 @@ static int test_init_dont_parse_argv(void)
}


static void set_all_env_vars(void)
static void set_most_env_vars(void)
{
putenv("PYTHONHASHSEED=42");
putenv("PYTHONMALLOC=malloc");
Expand All @@ -585,13 +585,15 @@ static void set_all_env_vars(void)
putenv("PYTHONNOUSERSITE=1");
putenv("PYTHONFAULTHANDLER=1");
putenv("PYTHONIOENCODING=iso8859-1:replace");
/* FIXME: test PYTHONWARNINGS */
/* FIXME: test PYTHONEXECUTABLE */
/* FIXME: test PYTHONHOME */
/* FIXME: test PYTHONDEBUG */
/* FIXME: test PYTHONDUMPREFS */
/* FIXME: test PYTHONCOERCECLOCALE */
/* FIXME: test PYTHONPATH */
}


static void set_all_env_vars(void)
{
set_most_env_vars();

putenv("PYTHONWARNINGS=EnvVar");
putenv("PYTHONPATH=/my/path");
}


Expand All @@ -609,7 +611,6 @@ static int test_init_env(void)

static void set_all_env_vars_dev_mode(void)
{
set_all_env_vars();
putenv("PYTHONMALLOC=");
putenv("PYTHONFAULTHANDLER=");
putenv("PYTHONDEVMODE=1");
Expand Down