Skip to content

Commit

Permalink
Issue python#7080: locale.strxfrm() raises a MemoryError on 64-bit no…
Browse files Browse the repository at this point in the history
…n-Windows

platforms, and assorted locale fixes by Derk Drukker.
  • Loading branch information
pitrou committed Oct 19, 2009
1 parent 86906f7 commit 6a448d4
Show file tree
Hide file tree
Showing 5 changed files with 52 additions and 13 deletions.
3 changes: 1 addition & 2 deletions Lib/locale.py
Original file line number Diff line number Diff line change
Expand Up @@ -578,14 +578,13 @@ def getpreferredencoding(do_setlocale = True):
# returning nothing will crash the
# interpreter.
result = 'UTF-8'

setlocale(LC_CTYPE, oldloc)
return result
else:
result = nl_langinfo(CODESET)
if not result and sys.platform == 'darwin':
# See above for explanation
result = 'UTF-8'
return result


### Database
Expand Down
49 changes: 40 additions & 9 deletions Lib/test/test_locale.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,10 @@

def get_enUS_locale():
global enUS_locale
if sys.platform == 'darwin':
raise unittest.SkipTest("Locale support on MacOSX is minimal")
if sys.platform.startswith("win"):
tlocs = ("En", "English")
else:
tlocs = ("en_US.UTF-8", "en_US.US-ASCII", "en_US")
tlocs = ("en_US.UTF-8", "en_US.ISO8859-1", "en_US.US-ASCII", "en_US")
oldlocale = locale.setlocale(locale.LC_NUMERIC)
for tloc in tlocs:
try:
Expand Down Expand Up @@ -309,6 +307,39 @@ def test_currency(self):
grouping=True, international=True)


class TestCollation(unittest.TestCase):
# Test string collation functions

def test_strcoll(self):
self.assertLess(locale.strcoll('a', 'b'), 0)
self.assertEqual(locale.strcoll('a', 'a'), 0)
self.assertGreater(locale.strcoll('b', 'a'), 0)

def test_strxfrm(self):
self.assertLess(locale.strxfrm('a'), locale.strxfrm('b'))


class TestEnUSCollation(BaseLocalizedTest, TestCollation):
# Test string collation functions with a real English locale

locale_type = locale.LC_ALL

def setUp(self):
BaseLocalizedTest.setUp(self)
enc = codecs.lookup(locale.getpreferredencoding(False) or 'ascii').name
if enc not in ('utf-8', 'iso8859-1', 'cp1252'):
raise unittest.SkipTest('encoding not suitable')
if enc != 'iso8859-1' and (sys.platform == 'darwin' or
sys.platform.startswith('freebsd')):
raise unittest.SkipTest('wcscoll/wcsxfrm have known bugs')

def test_strcoll_with_diacritic(self):
self.assertLess(locale.strcoll('à', 'b'), 0)

def test_strxfrm_with_diacritic(self):
self.assertLess(locale.strxfrm('à'), locale.strxfrm('b'))


class TestMiscellaneous(unittest.TestCase):
def test_getpreferredencoding(self):
# Invoke getpreferredencoding to make sure it does not cause exceptions.
Expand All @@ -317,11 +348,10 @@ def test_getpreferredencoding(self):
# If encoding non-empty, make sure it is valid
codecs.lookup(enc)

if hasattr(locale, "strcoll"):
def test_strcoll_3303(self):
# test crasher from bug #3303
self.assertRaises(TypeError, locale.strcoll, "a", None)
self.assertRaises(TypeError, locale.strcoll, b"a", None)
def test_strcoll_3303(self):
# test crasher from bug #3303
self.assertRaises(TypeError, locale.strcoll, "a", None)
self.assertRaises(TypeError, locale.strcoll, b"a", None)


def test_main():
Expand All @@ -331,6 +361,7 @@ def test_main():
TestEnUSNumberFormatting,
TestCNumberFormatting,
TestFrFRNumberFormatting,
TestCollation
]
# SkipTest can't be raised inside unittests, handle it manually instead
try:
Expand All @@ -339,7 +370,7 @@ def test_main():
if verbose:
print("Some tests will be disabled: %s" % e)
else:
tests += [TestNumberFormatting]
tests += [TestNumberFormatting, TestEnUSCollation]
run_unittest(*tests)

if __name__ == '__main__':
Expand Down
5 changes: 4 additions & 1 deletion Misc/NEWS
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,9 @@ C-API
Library
-------

- Issue #7080: locale.strxfrm() raises a MemoryError on 64-bit non-Windows
platforms, and assorted locale fixes by Derk Drukker.

- Issue #5833: Fix extra space character in readline completion with the
GNU readline library version 6.0.

Expand All @@ -118,7 +121,7 @@ Library
is too large to fit in the current precision.

- Issue #6236, #6348: Fix various failures in the I/O library under AIX
and other platforms, when using a non-gcc compiler. Patch by egreen.
and other platforms, when using a non-gcc compiler. Patch by Derk Drukker.

- Issue #4606: Passing 'None' if ctypes argtype is set to POINTER(...)
does now always result in NULL.
Expand Down
3 changes: 2 additions & 1 deletion Modules/_localemodule.c
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ This software comes with no warranty. Use at your own risk.
******************************************************************/

#define PY_SSIZE_T_CLEAN
#include "Python.h"

#include <stdio.h>
Expand Down Expand Up @@ -315,7 +316,7 @@ PyLocale_strxfrm(PyObject* self, PyObject* args)
result = PyUnicode_FromWideChar(buf, n2);
exit:
if (buf) PyMem_Free(buf);
#ifdef HAVE_USABLE_WCHAR_T
#ifndef HAVE_USABLE_WCHAR_T
PyMem_Free(s);
#endif
return result;
Expand Down
5 changes: 5 additions & 0 deletions PC/pyconfig.h
Original file line number Diff line number Diff line change
Expand Up @@ -641,6 +641,11 @@ Py_NO_ENABLE_SHARED to find out. Also support MS_NO_COREDLL for b/w compat */
#define HAVE_WCSCOLL 1
#endif

/* Define to 1 if you have the `wcsxfrm' function. */
#ifndef MS_WINCE
#define HAVE_WCSXFRM 1
#endif

/* Define if you have the <dlfcn.h> header file. */
/* #undef HAVE_DLFCN_H */

Expand Down

0 comments on commit 6a448d4

Please sign in to comment.